Replace Google Drive with GitHub release assets

Thynix · Thynix · commit 04b57b6fd008 · 2014-09-27T20:57:17.000-04:00
Google Drive has caused some problems when hosting assets. It has blocked downloads when detecting them as viruses, and hit download limits. GitHub has different policies: At this time, we do not place any limits on the size of binary uploads nor the bandwidth used to deliver them. Via https://help.github.com/articles/distributing-large-binaries
diff --git a/.gitignore b/.gitignore
@@ -1,4 +1,4 @@
-# Google Drive API credentials
-/client_secrets.json
-# Generated links go Google Drive files. (upload-to-google-drive.py script)
-/google_drive_links
+__pycache__/
+
+# Generated by upload-assets
+/asset_urls
diff --git a/README.md b/README.md
@@ -73,11 +73,7 @@ slash. Surrounding the value with double quotes is optional.)
 * Dependencies such as plugins either built or
 [downloaded](https://github.com/freenet/fred/blob/next/src/freenet/pluginmanager/OfficialPlugins.java#L23),
 (listing of loadedFrom() CHKs) into FreenetReleased. Repositories using these files can be set up with symlinks.
-* That pyDrive is [set up](http://pythonhosted.org/PyDrive/quickstart.html#authentication).
-  (Already installed by `setup-release-environment`.) Note that this
-  requires setting a product name and email address on the "APIs & Auth" > "Consent Screen" page.
-  To avoid the application launching a browser as in the authentication directed by the quick start guide,
-  create an "installed application" OAuth client ID and change `googleDriveAuth` in `freenetrc` to `"cmdline"`.
+* A GitHub OAuth token with `public_repo` access set in `~/.freenetrc` under `gitHubOAuthToken`.
 * A jarsigner certificate. This can be a self-signed one, though once (or if) one exists for FPI one should use it. See [here](http://docs.oracle.com/javase/6/docs/technotes/tools/windows/keytool.html). For example: `keytool -genkeypair -keyalg RSA -sigalg SHA256withRSA -keysize 4096 -dname "cn=Robert Releaserson, o=The Freenet Project Inc, c=US" -alias freenet -storepass SomePassphrase -validity 365 -keystore ~/.keystore`
  * Set freenetrc `jarsignerAlias` and `jarsignerPassword` to the alias and store passphrase, respectively.
 * For the Java installer: [launch4j](http://sourceforge.net/projects/launch4j/)
@@ -137,7 +133,7 @@ If used with `--snapshot` inserts the Fred jar and signature into Freenet.
 
 5. `java -jar [location of released jars]/new_installer_offline_[buildnumber].jar` runs an installer. The release manager should test installing a node both with the Linux / OS X installer and the Windows one. It should be able to bootstrap successfully, access FProxy, and otherwise have no obvious problems.
 
-6. `upload-to-google-drive.py` uploads the jars and installers to Google Drive which serves the majority of downloads.
+6. `upload-assets` uploads the jars and installers to GitHub which serves the majority of downloads.
 
 7. `deploy-website`, when run from osprey, updates the website to point to the latest version as defined by the given `fred` repository. The script's `-u` switch updates both `fred` and `website`, so if one wants to avoid pulling in website changes as well it may be preferable to manually update the `fred` repository only, or use the `--force-*-id` options.
 
diff --git a/deploy-website b/deploy-website
@@ -148,18 +148,18 @@ popd
 #Determine version information.
 getBuildInfo
 
-#Read links to release files on Google Drive.
-if [ ! -e google_drive_links ]; then
-    echo "Missing Google Drive links. Please generate them:"
-    echo "python upload-to-google-drive.py $buildNumber --skip_upload"
+# Read URLs of release files.
+if [ ! -e asset_urls ]; then
+    echo "Missing asset URLs. Please generate them:"
+    echo "./upload-assets $buildNumber"
     exit 3
 fi
-source google_drive_links || exit
+source asset_urls || exit
 
-if [[ "$linkBuildNumber" -ne "$buildNumber" ]]; then
-    echo "The current build is $buildNumber but the links are for $linkBuildNumber"
-    echo "Please generate them again:"
-    echo "python upload-to-google-drive.py $buildNumber --skip_upload"
+if [[ "$assetBuildNumber" -ne "$buildNumber" ]]; then
+    echo "The current build is $buildNumber but the URLs are for $assetBuildNumber"
+    echo "Please generate them:"
+    echo "./upload-assets $buildNumber"
     exit 3
 fi
 
diff --git a/freenetrc-sample b/freenetrc-sample
@@ -44,12 +44,8 @@ useSshAgent="1"
 # Hostname to send updates to.
 targetHost="osprey.vm.bytemark.co.uk"
 
-# Google Drive upload authentication method.
-# "cmdline": print the URL of a page to be manually opened. This page
-#            gives a code to type/copy in.
-# "browser": open a browser with a URL that allows logging in. This
-#            avoids having to type in the code.
-googleDriveAuth="browser"
+# GitHub OAuth token with repo access for uploading assets.
+gitHubOAuthToken="token"
 
 # FCP host hostname / IP.
 fcpHost="127.0.0.1"
diff --git a/github_releases.py b/github_releases.py
@@ -0,0 +1,150 @@
+import json
+import http.client
+import ssl
+import os
+import mimetypes
+
+
+class GitHubError(Exception):
+    """
+    GitHub gave a response other than success. Check the response attribute
+    for details.
+    """
+    def __init__(self, response):
+        self.response = response
+
+
+class UnknownMIMETypeError(Exception):
+    """
+    A MIME type for the file could not be detected automatically.
+    """
+    pass
+
+
+class GitHubReleases(object):
+
+    def __init__(self, oauth_token, user_agent):
+        self.base_headers = {
+            'Accept': 'application/vnd.github.v3+json',
+            'Authorization': 'token {}'.format(oauth_token),
+            'User-Agent': user_agent,
+        }
+
+        context = ssl.create_default_context()
+
+        # TODO: Connect once at the start or for each request?
+        self.api_host = http.client.HTTPSConnection('api.github.com',
+                                                    context=context)
+        self.api_host.connect()
+
+        self.uploads_host = http.client.HTTPSConnection('uploads.github.com',
+                                                        context=context)
+        self.uploads_host.connect()
+
+    def create(self, owner, repo, tag):
+        """
+        Create a release from a tag in a repo.
+
+        Note that this is a small subset of the GitHub API v3 release creation
+        capabilities.
+
+        :param owner: The owner user or organization.
+        :param repo: The name of the repository.
+        :param tag: The tag to create a release from.
+        """
+        self.api_host.request("POST",
+                              "/repos/{}/{}/releases".format(owner, repo),
+                              json.dumps(
+                                  {
+                                      'tag_name': tag,
+                                      'target_commitish': None,
+                                      'name': None,
+                                      'body': None,
+                                      'draft': False,
+                                      'prerelease': False,
+                                  }), self.base_headers)
+        return self.__get_response(self.api_host)
+
+    def get(self, owner, repo, tag):
+        """
+        Get the release for the given tag.
+
+        :param owner: The owner user or organization.
+        :param repo: The name of the repository.
+        :param tag: The tag to find the release of.
+        :return: result for the release with that tag, or None if no such
+                 release.
+        """
+        self.api_host.request("GET",
+                              "/repos/{}/{}/releases".format(owner, repo, tag),
+                              headers=self.base_headers)
+        response = self.__get_response(self.api_host, expecting_code=200)
+
+        for release in response:
+            if release['tag_name'] == tag:
+                return release
+        else:
+            return None
+
+    def upload_asset(self, owner, repo, release_id, asset_path,
+                     asset_type=None):
+        """
+        Upload an asset to a release.
+
+        :param owner: The owner user or organization.
+        :param repo: The name of the repository.
+        :param release_id: ID of the release to upload to.
+        :param asset_path: Path to the file to upload.
+        :param asset_type: Optional MIME type of the asset. If None or not
+                           specified it will be detected.
+        :throws UnknownMimeTypeError:
+        """
+        filename = os.path.basename(asset_path)
+
+        if asset_type is None:
+            asset_type = mimetypes.guess_type(filename)[0]
+            if asset_type is None:
+                raise UnknownMIMETypeError()
+
+        headers = {'Content-Type': asset_type}
+        headers.update(self.base_headers)
+
+        self.uploads_host.request(
+            "POST",
+            "/repos/{}/{}/releases/{}/assets?name={}"
+            .format(owner, repo, release_id, filename),
+            open(asset_path, "rb"),
+            headers
+        )
+        return self.__get_response(self.uploads_host)
+
+    def list_assets(self, owner, repo, release_id):
+        """
+        List assets in the release.
+
+        :param owner: The owner user or organization.
+        :param repo: The name of the repository.
+        :param release_id: ID of the release to list assets from.
+        :returns: List of assets.
+        """
+        self.api_host.request(
+            "GET",
+            "/repos/{}/{}/releases/{}/assets"
+            .format(owner, repo, release_id),
+            headers=self.base_headers
+        )
+        return self.__get_response(self.api_host, expecting_code=200)
+
+    @staticmethod
+    def __get_response(host, expecting_code=201):
+        # TODO: Does GitHub _always_ give something that's valid JSON? What
+        # error handling to have here?
+        # TODO: What encoding?
+        response = host.getresponse()
+        body = response.read()
+        result = json.loads(body.decode("utf8"))
+
+        if response.code != expecting_code:
+            raise GitHubError(result)
+
+        return result
diff --git a/release-build b/release-build
@@ -34,7 +34,7 @@ echo Releasing build $buildNumber
 release-fred --build || exit 2
 release-installer || exit 3
 release-wininstaller --release || exit 4
-python upload-to-google-drive.py $buildNumber --release_directory "$releaseDir" --auth_type "$googleDriveAuth" || exit 5
+env GITHUB_OAUTH="$gitHubOAuthToken" upload-assets $buildNumber "$releaseDir" || exit
 # REDFLAG IT IS NEVER SAFE TO INSERT A BUILD ON A DEVELOPMENT NODE!!!!!
 echo Now please test the new build.
 echo Deploy the build only when:
diff --git a/remote-deploy-website b/remote-deploy-website
@@ -12,5 +12,5 @@ websiteID=$(cd "$websiteDir"; git rev-parse --verify master)
 # Both git describe and git rev-parse --verify will give a tag if it exists.
 echo Fred ID "$commitID"
 echo Website ID "$websiteID"
-scp google_drive_links "$targetHost:$remoteDeployPrefix"
+scp asset_urls "$targetHost:$remoteDeployPrefix"
 ssh $targetHost -- "cd $remoteDeployPrefix; ./deploy-website -u --force-website-id $websiteID --force-fred-id $commitID"
diff --git a/setup-release-environment b/setup-release-environment
@@ -8,16 +8,10 @@ source freenet-scripts-common || exit
 # For release management: rsync, coreutils, diffutils, gnupg, less, wget, perl, python, pip
 # For building the Windows installer: xvfb, wine, wine-gecko, mono-runtime
 # For avoiding repeated password entry: gnupg-agent openssh-client
-# For Google Drive upload: PyDrive
 # For finding the system certificates: httplib2.ca-certs-locater
 # For prompting for things like "2 weeks" in update-version.py: timeparser
 # It would be interesting to have the scripts repo without git, but might as well.
-sudo apt-get install git openjdk-6-jdk rsync ant coreutils gnupg diffutils less wget xvfb wine perl gnupg-agent openssh-client mono-runtime wine-gecko-1.4 netcat python python-pip || exit
-# Unlike 'sudo', 'sudo su' sets umask to the default for root, which
-# should be 0022 so that installed files are world-readable.
-# TODO: This doesn't actually seem to be the case when checking the umask explicitly: (sudo|) bash -c umask. Is using sudo su appropriate?
-# See http://stackoverflow.com/questions/11161776/pip-inconsistent-permissions-issues
-sudo su -c 'pip install PyDrive httplib2.ca-certs-locater timeparser' || exit
+sudo apt-get install git openjdk-6-jdk rsync ant coreutils gnupg diffutils less wget xvfb wine perl gnupg-agent openssh-client mono-runtime wine-gecko-1.4 netcat python python3 || exit
 
 mkdir -p "$releaseDir/dependencies"
 wget "https://people.apache.org/~ebourg/jsign/jsign-1.2.jar" -O "$releaseDir/dependencies/jsign-1.2.jar"
diff --git a/upload-assets b/upload-assets
@@ -0,0 +1,100 @@
+#!/usr/bin/env python3
+import argparse
+import os
+
+import github_releases
+
+parser = argparse.ArgumentParser(description="""
+Upload a release to GitHub: the .jar, the source archive (so the signatures
+will match it), the Java installer, and the Windows installer. If no directory
+is specified it will read existing assets.
+
+It assumes the build tag is 'build' followed by the build number
+zero-padded to 5 characters.
+""")
+parser.add_argument('build_number', help='The build number to upload.',
+                    type=int)
+parser.add_argument('release_directory',
+                    help='Path to the FreenetReleased directory containing '
+                         'the assets to upload.', nargs='?', default=None)
+parser.add_argument('--out', default='asset_urls',
+                    help='File to output URLs to')
+
+args = parser.parse_args()
+build_tag = 'build{0:05d}'.format(args.build_number)
+
+# TODO: Allow resuming: avoid clobbering existing assets when uploading.
+# TODO: Nicer error on missing environment variable.
+# TODO: Bind owner and repo to GitHubReleases instance? They're in every call.
+# TODO: Restructure the body of this into a module?
+# TODO: The name 'upload-assets' makes the non-upload behavior surprising.
+#  What's a better name for this script?
+conn = github_releases.GitHubReleases(os.environ['GITHUB_OAUTH'],
+                                      'Freenet-Release-Uploader')
+
+# Create a release at the tag, or get its ID if one already exists.
+try:
+    release = conn.create('freenet', 'fred', build_tag)
+    release_id = release['id']
+    print('Created release {}'.format(release_id))
+except github_releases.GitHubError as e:
+    # TODO: Avoid hardcoding here?
+    if e.response['errors'][0]['code'] == 'already_exists':
+        release_id = conn.get('freenet', 'fred', build_tag)['id']
+        print('Found release {}'.format(release_id))
+    else:
+        raise
+
+files = {
+    'jar': 'freenet-{0}.jar'.format(build_tag),
+    'source': 'freenet-{0}-source.tar.bz2'.format(build_tag),
+    'java_installer': 'new_installer_offline_{0}.jar'.format(args.build_number),
+    'windows_installer': 'FreenetInstaller-{0}.exe'.format(args.build_number),
+}
+urls = {}
+
+# Upload if the release directory is specified, otherwise find existing assets.
+if args.release_directory is not None:
+    for name, file in files.items():
+        path = os.path.join(args.release_directory, file)
+
+        print("Uploading asset {}".format(file))
+        result = conn.upload_asset("freenet", "fred", release_id, path)
+
+        urls[name] = result['url']
+else:
+    result = conn.list_assets("freenet", "fred", release_id)
+
+    assets = {}
+    for asset in result:
+        assets[asset['name']] = asset['browser_download_url']
+
+    for name, file in files.items():
+        if file not in assets:
+            print('Missing asset: {}'.format(file))
+            exit(1)
+
+        print("Found asset {}".format(file))
+        urls[name + "_url"] = assets[file]
+
+# Each file must have a URL.
+assert len(files) == len(urls)
+
+# Output URLs for use in substitution.
+# TODO: Verbosity vs readability with something like this?
+with open(args.out, 'w') as output:
+    total = files.copy()
+    total.update(urls)
+
+    output.write("""\
+# Generated for build {build_number} with upload-assets
+assetBuildNumber={build_number}
+FREENET_MAIN_JAR_URL={jar_url}
+FREENET_SOURCE_URL={source_url}
+FREENET_INSTALLER_URL={java_installer_url}
+FREENET_WINDOWS_INSTALLER_URL={windows_installer_url}
+FREENET_MAIN_JAR_SIG_URL=https://downloads.freenetproject.org/alpha/{jar}.sig
+FREENET_SOURCE_SIG_URL=https://downloads.freenetproject.org/alpha/{source}.sig
+FREENET_INSTALLER_SIG_URL=https://downloads.freenetproject.org/alpha/installer/{java_installer}.sig
+FREENET_WINDOWS_INSTALLER_SIG_URL=https://downloads.freenetproject.org/alpha/installer/{windows_installer}.sig
+""".format(build_number=args.build_number, **total))
diff --git a/upload-to-google-drive.py b/upload-to-google-drive.py
