From f5085d3c1fd2a88bd2993525f5d235b9b7334b96 Mon Sep 17 00:00:00 2001 From: meows Date: Wed, 4 Mar 2020 06:41:09 -0600 Subject: [PATCH 1/3] Init Github Action verbatim from etclabscore/discv4-dns-lists Signed-off-by: meows --- .ci/crawl.sh | 12 +++++ .ci/deploy.sh | 20 +++++++++ .ci/deps.sh | 38 ++++++++++++++++ .ci/filter_and_sign.sh | 39 +++++++++++++++++ .github/workflows/crawl.yml | 87 +++++++++++++++++++++++++++++++++++++ 5 files changed, 196 insertions(+) create mode 100755 .ci/crawl.sh create mode 100755 .ci/deploy.sh create mode 100755 .ci/deps.sh create mode 100755 .ci/filter_and_sign.sh create mode 100644 .github/workflows/crawl.yml diff --git a/.ci/crawl.sh b/.ci/crawl.sh new file mode 100755 index 0000000000..45b6560667 --- /dev/null +++ b/.ci/crawl.sh @@ -0,0 +1,12 @@ +#!/bin/sh + +devp2p discv4 crawl -timeout "$ETH_DNS_DISCV4_CRAWLTIME" all.json + +git add all.json + +ETH_DNS_DISCV4_KEY_PUBLICINFO="$(cat $ETH_DNS_DISCV4_KEYPASS_PATH | ethkey inspect $ETH_DNS_DISCV4_KEY_PATH | grep -E '(Addr|Pub)')" +git -c user.name="meows" -c user.email='b5c6@protonmail.com' commit --author 'crawler <>' -m "ci update (all.json) $GITHUB_RUN_ID:$GITHUB_RUN_NUMBER + +Crawltime: $ETH_DNS_DISCV4_CRAWLTIME + +$ETH_DNS_DISCV4_KEY_PUBLICINFO" \ No newline at end of file diff --git a/.ci/deploy.sh b/.ci/deploy.sh new file mode 100755 index 0000000000..e1b9917b93 --- /dev/null +++ b/.ci/deploy.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +set -e + +proto_groups=(all les) + +for network in "$@"; do + + echo "Deploy: $network" + + for p in "${proto_groups[@]}"; do + echo -n "Deploy: ${p}.${network}.${ETH_DNS_DISCV4_PARENT_DOMAIN}" + + # Ensure that we actually have a nodeset to deploy to DNS. + [[ ! -d ${p}.${network}.${ETH_DNS_DISCV4_PARENT_DOMAIN} ]] || [[ ! -f ${p}.${network}.${ETH_DNS_DISCV4_PARENT_DOMAIN}/nodes.json ]] && { echo " | DNE, skipping"; continue; } + + echo + devp2p dns to-cloudflare --zoneid "$ETH_DNS_CLOUDFLARE_ZONEID" "${p}.${network}.${ETH_DNS_DISCV4_PARENT_DOMAIN}" + done +done diff --git a/.ci/deps.sh b/.ci/deps.sh new file mode 100755 index 0000000000..89af252d7f --- /dev/null +++ b/.ci/deps.sh @@ -0,0 +1,38 @@ +#!/bin/sh + +set -e + +# Check programs we depend on. +command -v devp2p >/dev/null 2>&1 && echo "OK: devp2p command in PATH" || { echo "Please install devp2p"; exit 1; } +command -v ethkey >/dev/null 2>&1 && echo "OK: ethkey command in PATH" || { echo "Please install ethkey"; exit 1; } + +# Check that we have key and keypass file. +if [ ! -f $ETH_DNS_DISCV4_KEY_PATH ] || [ ! -f $ETH_DNS_DISCV4_KEYPASS_PATH ]; then + echo " +No key found at key file path or no password file found at ${ETH_DNS_DISCV4_KEYPASS_PATH}. +Use 'ethkey generate ${ETH_DNS_DISCV4_KEY_PATH}' +Save the password in plaintext in ${ETH_DNS_DISCV4_KEYPASS_PATH} +" + exit 1 +else + echo "OK: Key and password file exist." +fi + +# Check that we have deploy variables set. +if [ -z $CLOUDFLARE_API_TOKEN ]; then + echo "Missing CLOUDFLARE_API_TOKEN env var" + exit 1 +else + echo "OK: environment variable CLOUDFLARE_API_TOKEN is not empty." +fi + +# I'm not sure why I couldn't get devp2p to work without using the --zoneid flag; kept getting 403 bad perms. +# Using the flag seems to fix it, and this var gets set as the value for that flag. +# It's associated with the specific domain name that Cloudflare is managing and that we want to deploy to. +if [ -z $ETH_DNS_CLOUDFLARE_ZONEID ]; then + echo "Missing ETH_DNS_CLOUDFLARE_ZONEID env var" + exit 1 +else + echo "OK: environment variable ETH_DNS_CLOUDFLARE_ZONEID is not empty." +fi + diff --git a/.ci/filter_and_sign.sh b/.ci/filter_and_sign.sh new file mode 100755 index 0000000000..17af5b3c3d --- /dev/null +++ b/.ci/filter_and_sign.sh @@ -0,0 +1,39 @@ +#!/bin/bash + +set -e + +proto_groups=(all les) + +for network in "$@"; do + + echo "Filter: $network" + + mkdir -p "all.${network}.${ETH_DNS_DISCV4_PARENT_DOMAIN}" + devp2p nodeset filter all.json -eth-network "$network" >"all.${network}.${ETH_DNS_DISCV4_PARENT_DOMAIN}/nodes.json" + + mkdir -p "les.${network}.${ETH_DNS_DISCV4_PARENT_DOMAIN}" + devp2p nodeset filter all.json -les-server -eth-network "$network" >"les.${network}.${ETH_DNS_DISCV4_PARENT_DOMAIN}/nodes.json" + + + echo "Sign: $network" + + for p in "${proto_groups[@]}"; do + echo -n "Sign: ${p}.${network}.${ETH_DNS_DISCV4_PARENT_DOMAIN}" + + # Ensure that we actually have a nodeset to sign. + [ ! -d ${p}.${network}.${ETH_DNS_DISCV4_PARENT_DOMAIN} ] || [ ! -f ${p}.${network}.${ETH_DNS_DISCV4_PARENT_DOMAIN}/nodes.json ] && { echo " | DNE, skipping"; continue; } + + echo + cat "${ETH_DNS_DISCV4_KEYPASS_PATH}" | devp2p dns sign "${p}.${network}.${ETH_DNS_DISCV4_PARENT_DOMAIN}" "${ETH_DNS_DISCV4_KEY_PATH}" && echo "OK" + + git add "${p}.${network}.${ETH_DNS_DISCV4_PARENT_DOMAIN}" + done + + ETH_DNS_DISCV4_KEY_PUBLICINFO="$(cat $ETH_DNS_DISCV4_KEYPASS_PATH | ethkey inspect $ETH_DNS_DISCV4_KEY_PATH | grep -E '(Addr|Pub)')" + git -c user.name="meows" -c user.email='b5c6@protonmail.com' commit --author "crawler <>" -m "ci update ($network) $GITHUB_RUN_ID:$GITHUB_RUN_NUMBER + +Crawltime: $ETH_DNS_DISCV4_CRAWLTIME + +$ETH_DNS_DISCV4_KEY_PUBLICINFO" + +done \ No newline at end of file diff --git a/.github/workflows/crawl.yml b/.github/workflows/crawl.yml new file mode 100644 index 0000000000..efe3d81638 --- /dev/null +++ b/.github/workflows/crawl.yml @@ -0,0 +1,87 @@ +name: Discv4 Crawl and DNS Update +on: + schedule: + - cron: '0 */2 * * *' +jobs: + build: + if: github.repository == 'etclabscore/discv4-dns-lists' + name: Discv4-DNS-Crawler + runs-on: ubuntu-latest + env: + ETH_DNS_DISCV4_CRAWLTIME: 30m + ETH_DNS_DISCV4_PARENT_DOMAIN: blockd.info + ETH_DNS_DISCV4_KEY_PATH: ./secrets/dnskey.json + ETH_DNS_DISCV4_KEYPASS_PATH: ./secrets/dnskey_password.txt + CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} + ETH_DNS_CLOUDFLARE_ZONEID: ${{ secrets.ETH_DNS_CLOUDFLARE_ZONEID }} + ETH_DNS_DISCV4_KEY: ${{ secrets.ETH_DNS_DISCV4_KEY }} + ETH_DNS_DISCV4_KEYPASS: ${{ secrets.ETH_DNS_DISCV4_KEYPASS }} + + steps: + - name: Set up Go + uses: actions/setup-go@v2-beta + with: + go-version: 1.13.8 + id: go + + - run: go version + + - name: Check out code + uses: actions/checkout@v2 + with: + ref: etccore + token: ${{ secrets.PAT_REPO_MEOWSBITS }} + + - uses: actions/checkout@v2 + with: + ref: etccore + token: ${{ secrets.PAT_REPO_MEOWSBITS }} + - name: Checkout submodules + shell: bash + run: | + # If your submodules are configured to use SSH instead of HTTPS please uncomment the following line + # git config --global url."https://github.com/".insteadOf "git@github.com:" + auth_header="$(git config --local --get http.https://github.com/.extraheader)" + git submodule sync --recursive + git -c "http.extraheader=$auth_header" -c protocol.version=2 submodule update --init --force --depth=1 + cd core-geth + git fetch origin --tags + latest_tag="$(git describe --abbrev=0)" + git checkout ${latest_tag} + make all + echo "::add-path::$(pwd)/build/bin" + cd .. + git diff --quiet || { git add core-geth && git -c user.name='meows' -c user.email='b5c6@protonmail.com' commit --author='crawler <>' -m "ci update (core-geth:${latest_tag}) $GITHUB_RUN_ID:$GITHUB_RUN_NUMBER"; } + + - name: Setup secrets + run: | + mkdir secrets + echo "$ETH_DNS_DISCV4_KEY" > "$ETH_DNS_DISCV4_KEY_PATH" + echo "$ETH_DNS_DISCV4_KEYPASS" > "$ETH_DNS_DISCV4_KEYPASS_PATH" + + - name: Check env and secrets + run: | + ./.ci/deps.sh + export ETH_DNS_DISCV4_KEY_PUBLICINFO="$(cat $ETH_DNS_DISCV4_KEYPASS_PATH | ethkey inspect $ETH_DNS_DISCV4_KEY_PATH | grep -E '(Addr|Pub)')" + echo "$ETH_DNS_DISCV4_KEY_PUBLICINFO" + + - name: Crawl + run: | + ./.ci/crawl.sh + + - name: Filter and sign + run: | + ./.ci/filter_and_sign.sh classic kotti mordor + + - name: Deploy to DNS + run: | + ./.ci/deploy.sh classic kotti mordor + + - name: Push + env: + GITHUB_PAT: ${{ secrets.PAT_REPO_MEOWSBITS }} + run: | + git config --local user.name 'meows' + git config --local user.email 'b5c6@protonmail.com' + git remote set-url origin https://meowsbits:${GITHUB_PAT}@github.com/${GITHUB_REPOSITORY}.git + git push origin etccore \ No newline at end of file From 8bc7147db507f311ab4bb919578a2dc3e45eb2b2 Mon Sep 17 00:00:00 2001 From: meows Date: Wed, 4 Mar 2020 06:42:21 -0600 Subject: [PATCH 2/3] include missing .gitignore Signed-off-by: meows --- .gitignore | 1 + 1 file changed, 1 insertion(+) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000000..4bd922a161 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +secrets/ From a401a89fa5254c711b9966fca406acf9a5a1ea7c Mon Sep 17 00:00:00 2001 From: meows Date: Wed, 4 Mar 2020 06:58:22 -0600 Subject: [PATCH 3/3] Revise to be ETH friendly Signed-off-by: meows --- .ci/crawl.sh | 11 +------- .ci/filter_and_sign.sh | 10 +------ .github/workflows/crawl.yml | 56 +++++++++++++------------------------ 3 files changed, 22 insertions(+), 55 deletions(-) diff --git a/.ci/crawl.sh b/.ci/crawl.sh index 45b6560667..23247d4336 100755 --- a/.ci/crawl.sh +++ b/.ci/crawl.sh @@ -1,12 +1,3 @@ #!/bin/sh -devp2p discv4 crawl -timeout "$ETH_DNS_DISCV4_CRAWLTIME" all.json - -git add all.json - -ETH_DNS_DISCV4_KEY_PUBLICINFO="$(cat $ETH_DNS_DISCV4_KEYPASS_PATH | ethkey inspect $ETH_DNS_DISCV4_KEY_PATH | grep -E '(Addr|Pub)')" -git -c user.name="meows" -c user.email='b5c6@protonmail.com' commit --author 'crawler <>' -m "ci update (all.json) $GITHUB_RUN_ID:$GITHUB_RUN_NUMBER - -Crawltime: $ETH_DNS_DISCV4_CRAWLTIME - -$ETH_DNS_DISCV4_KEY_PUBLICINFO" \ No newline at end of file +devp2p discv4 crawl -timeout "$ETH_DNS_DISCV4_CRAWLTIME" all.json \ No newline at end of file diff --git a/.ci/filter_and_sign.sh b/.ci/filter_and_sign.sh index 17af5b3c3d..9d31353484 100755 --- a/.ci/filter_and_sign.sh +++ b/.ci/filter_and_sign.sh @@ -14,7 +14,7 @@ for network in "$@"; do mkdir -p "les.${network}.${ETH_DNS_DISCV4_PARENT_DOMAIN}" devp2p nodeset filter all.json -les-server -eth-network "$network" >"les.${network}.${ETH_DNS_DISCV4_PARENT_DOMAIN}/nodes.json" - + echo "Sign: $network" for p in "${proto_groups[@]}"; do @@ -26,14 +26,6 @@ for network in "$@"; do echo cat "${ETH_DNS_DISCV4_KEYPASS_PATH}" | devp2p dns sign "${p}.${network}.${ETH_DNS_DISCV4_PARENT_DOMAIN}" "${ETH_DNS_DISCV4_KEY_PATH}" && echo "OK" - git add "${p}.${network}.${ETH_DNS_DISCV4_PARENT_DOMAIN}" done - - ETH_DNS_DISCV4_KEY_PUBLICINFO="$(cat $ETH_DNS_DISCV4_KEYPASS_PATH | ethkey inspect $ETH_DNS_DISCV4_KEY_PATH | grep -E '(Addr|Pub)')" - git -c user.name="meows" -c user.email='b5c6@protonmail.com' commit --author "crawler <>" -m "ci update ($network) $GITHUB_RUN_ID:$GITHUB_RUN_NUMBER - -Crawltime: $ETH_DNS_DISCV4_CRAWLTIME - -$ETH_DNS_DISCV4_KEY_PUBLICINFO" done \ No newline at end of file diff --git a/.github/workflows/crawl.yml b/.github/workflows/crawl.yml index efe3d81638..7961a9c02b 100644 --- a/.github/workflows/crawl.yml +++ b/.github/workflows/crawl.yml @@ -4,12 +4,12 @@ on: - cron: '0 */2 * * *' jobs: build: - if: github.repository == 'etclabscore/discv4-dns-lists' + if: github.repository == 'ethereum/discv4-dns-lists' name: Discv4-DNS-Crawler runs-on: ubuntu-latest env: ETH_DNS_DISCV4_CRAWLTIME: 30m - ETH_DNS_DISCV4_PARENT_DOMAIN: blockd.info + ETH_DNS_DISCV4_PARENT_DOMAIN: ethdisco.net ETH_DNS_DISCV4_KEY_PATH: ./secrets/dnskey.json ETH_DNS_DISCV4_KEYPASS_PATH: ./secrets/dnskey_password.txt CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} @@ -17,6 +17,7 @@ jobs: ETH_DNS_DISCV4_KEY: ${{ secrets.ETH_DNS_DISCV4_KEY }} ETH_DNS_DISCV4_KEYPASS: ${{ secrets.ETH_DNS_DISCV4_KEYPASS }} + steps: - name: Set up Go uses: actions/setup-go@v2-beta @@ -28,30 +29,11 @@ jobs: - name: Check out code uses: actions/checkout@v2 - with: - ref: etccore - token: ${{ secrets.PAT_REPO_MEOWSBITS }} - - uses: actions/checkout@v2 - with: - ref: etccore - token: ${{ secrets.PAT_REPO_MEOWSBITS }} - - name: Checkout submodules - shell: bash + - name: Install dependencies run: | - # If your submodules are configured to use SSH instead of HTTPS please uncomment the following line - # git config --global url."https://github.com/".insteadOf "git@github.com:" - auth_header="$(git config --local --get http.https://github.com/.extraheader)" - git submodule sync --recursive - git -c "http.extraheader=$auth_header" -c protocol.version=2 submodule update --init --force --depth=1 - cd core-geth - git fetch origin --tags - latest_tag="$(git describe --abbrev=0)" - git checkout ${latest_tag} - make all - echo "::add-path::$(pwd)/build/bin" - cd .. - git diff --quiet || { git add core-geth && git -c user.name='meows' -c user.email='b5c6@protonmail.com' commit --author='crawler <>' -m "ci update (core-geth:${latest_tag}) $GITHUB_RUN_ID:$GITHUB_RUN_NUMBER"; } + go get -u github.com/ethereum/go-ethereum/cmd/devp2p + go get -u github.com/ethereum/go-ethereum/cmd/ethkey - name: Setup secrets run: | @@ -62,8 +44,6 @@ jobs: - name: Check env and secrets run: | ./.ci/deps.sh - export ETH_DNS_DISCV4_KEY_PUBLICINFO="$(cat $ETH_DNS_DISCV4_KEYPASS_PATH | ethkey inspect $ETH_DNS_DISCV4_KEY_PATH | grep -E '(Addr|Pub)')" - echo "$ETH_DNS_DISCV4_KEY_PUBLICINFO" - name: Crawl run: | @@ -71,17 +51,21 @@ jobs: - name: Filter and sign run: | - ./.ci/filter_and_sign.sh classic kotti mordor + ./.ci/filter_and_sign.sh mainnet ropsten rinkeby goerli - - name: Deploy to DNS + - name: Commit and Push + env: + GITHUB_USER: FIXME + GITHUB_PAT: FIXME run: | - ./.ci/deploy.sh classic kotti mordor + git config --local user.name 'crawler' + git config --local user.email 'noreply@users.noreply.github.com' + git add all* les* + git commit --author 'crawler <>' -m "automatic update: crawl time ${ETH_DNS_DISCV4_CRAWLTIME} ci ${GITHUB_RUN_ID}:${GITHUB_RUN_NUMBER}" + git remote set-url origin https://${GITHUB_USER}:${GITHUB_PAT}@github.com/${GITHUB_REPOSITORY}.git + git push origin master - - name: Push - env: - GITHUB_PAT: ${{ secrets.PAT_REPO_MEOWSBITS }} + - name: Deploy to DNS run: | - git config --local user.name 'meows' - git config --local user.email 'b5c6@protonmail.com' - git remote set-url origin https://meowsbits:${GITHUB_PAT}@github.com/${GITHUB_REPOSITORY}.git - git push origin etccore \ No newline at end of file + ./.ci/deploy.sh mainnet ropsten rinkeby goerli +