rewrite dev/releases/update_website.py, cleanup devtools (#5793)

* rewrite dev/releases/update_website.py to be compatible with new GapWWW
* remove global variables in utils_github
* move stuff out of utils.py that has only one user

Author: fingolfin

dev/releases/make_archives.py

@@ -24,15 +24,13 @@
 import subprocess
 import sys
 import tarfile
+from typing import List, Optional
 
 from utils import (
     download_with_sha256,
     error,
-    get_makefile_var,
     notice,
     patchfile,
-    run_with_log,
-    safe_git_fetch_tags,
     verify_command_available,
     verify_git_clean,
     verify_git_repo,
@@ -43,6 +41,27 @@
 if sys.version_info < (3, 6):
     error("Python 3.6 or newer is required")
 
+
+# helper for extracting values of variables set in the GAP Makefiles.rules
+def get_makefile_var(var: str) -> str:
+    res = subprocess.run(["make", f"print-{var}"], check=True, capture_output=True)
+    kv = res.stdout.decode("ascii").strip().split("=")
+    assert len(kv) == 2
+    assert kv[0] == var
+    return kv[1]
+
+
+# Run what ever <args> command and create appropriate log file
+def run_with_log(args: List[str], name: str, msg: Optional[str] = None) -> None:
+    if not msg:
+        msg = name
+    with open("../" + name + ".log", "w", encoding="utf-8") as fp:
+        try:
+            subprocess.run(args, check=True, stdout=fp, stderr=fp)
+        except subprocess.CalledProcessError:
+            error(msg + " failed. See " + name + ".log.")
+
+
 notice("Checking prerequisites")
 verify_command_available("curl")
 verify_command_available("git")
@@ -52,7 +71,11 @@
 verify_git_clean()
 
 # fetch tags, so we can properly detect
-safe_git_fetch_tags()
+try:
+    subprocess.run(["git", "fetch", "--tags"], check=True)
+except subprocess.CalledProcessError:
+    error("failed to fetch tags, you may have to do \n" + "git fetch --tags -f")
+
 
 # Creating tmp directory
 tmpdir = os.getcwd() + "/tmp"

dev/releases/make_github_release.py

@@ -8,13 +8,10 @@
 ##
 ##  SPDX-License-Identifier: GPL-2.0-or-later
 ##
-##  This script makes a github release and uploads all tar balls as assets.
-##  The name of the target repository CURRENT_REPO_NAME is defined in
-##  utils.py.
-##
-##  If we do import * from utils, then initialize_github can't overwrite the
-##  global CURRENT_REPO variables.
+##  This script makes a github release and uploads all tarballs as assets.
 ##
+import re
+import subprocess
 import sys
 
 import utils
@@ -24,26 +21,65 @@
 if len(sys.argv) != 3:
     error("usage: " + sys.argv[0] + " <tag_name> <path_to_release>")
 
+
+def is_possible_gap_release_tag(tag: str) -> bool:
+    return re.fullmatch(r"v[1-9]+\.[0-9]+\.[0-9]+(-.+)?", tag) is not None
+
+
+def verify_is_possible_gap_release_tag(tag: str) -> None:
+    if not is_possible_gap_release_tag(tag):
+        error(f"{tag} does not look like the tag of a GAP release version")
+
+
+# lightweight vs annotated
+# https://stackoverflow.com/questions/40479712/how-can-i-tell-if-a-given-git-tag-is-annotated-or-lightweight#40499437
+def is_annotated_git_tag(tag: str) -> bool:
+    res = subprocess.run(
+        ["git", "for-each-ref", "refs/tags/" + tag],
+        capture_output=True,
+        text=True,
+        check=False,
+    )
+    return res.returncode == 0 and res.stdout.split()[1] == "tag"
+
+
+def check_git_tag_for_release(tag: str) -> None:
+    if not is_annotated_git_tag(tag):
+        error(f"There is no annotated tag {tag}")
+    # check that tag points to HEAD
+    tag_commit = subprocess.run(
+        ["git", "rev-parse", tag + "^{}"], check=True, capture_output=True, text=True
+    ).stdout.strip()
+    head = subprocess.run(
+        ["git", "rev-parse", "HEAD"], check=True, capture_output=True, text=True
+    ).stdout.strip()
+    if tag_commit != head:
+        error(
+            f"The tag {tag} does not point to the current commit {head} but"
+            + f" instead points to {tag_commit}"
+        )
+
+
 TAG_NAME = sys.argv[1]
 PATH_TO_RELEASE = sys.argv[2]
 VERSION = TAG_NAME[1:]  # strip 'v' prefix
 
 utils.verify_git_clean()
-utils.verify_is_possible_gap_release_tag(TAG_NAME)
-utils_github.initialize_github()
+verify_is_possible_gap_release_tag(TAG_NAME)
+repo = utils_github.initialize_github()
 
-# Error if the tag TAG_NAME hasn't been pushed to CURRENT_REPO yet.
-if not any(tag.name == TAG_NAME for tag in utils_github.CURRENT_REPO.get_tags()):
-    error(f"Repository {utils_github.CURRENT_REPO_NAME} has no tag '{TAG_NAME}'")
+# Error if the tag TAG_NAME hasn't been pushed out yet.
+if not any(tag.name == TAG_NAME for tag in repo.get_tags()):
+    error(f"Repository {repo.full_name} has no tag '{TAG_NAME}'")
 
 # make sure that TAG_NAME
 # - exists
 # - is an annotated tag
 # - points to current HEAD
-utils.check_git_tag_for_release(TAG_NAME)
+check_git_tag_for_release(TAG_NAME)
 
 # Error if this release has been already created on GitHub
-if any(r.tag_name == TAG_NAME for r in utils_github.CURRENT_REPO.get_releases()):
+if any(r.tag_name == TAG_NAME for r in repo.get_releases()):
     error(f"Github release with tag '{TAG_NAME}' already exists!")
 
 # Create release
@@ -52,9 +88,7 @@
     + f"[CHANGES.md](https://github.com/gap-system/gap/blob/{TAG_NAME}/CHANGES.md) file."
 )
 notice(f"Creating release {TAG_NAME}")
-RELEASE = utils_github.CURRENT_REPO.create_git_release(
-    TAG_NAME, TAG_NAME, RELEASE_NOTE, prerelease=True
-)
+RELEASE = repo.create_git_release(TAG_NAME, TAG_NAME, RELEASE_NOTE, prerelease=True)
 
 with utils.working_directory(PATH_TO_RELEASE):
     manifest_filename = "MANIFEST"

dev/releases/release_notes.py

@@ -29,14 +29,21 @@
 from typing import Any, Dict, List, TextIO
 
 import requests
-from utils import download_with_sha256, error, is_existing_tag, notice, warning
+from utils import download_with_sha256, error, notice, warning
 
 
 def usage(name: str) -> None:
     print(f"Usage: `{name} NEWVERSION`")
     sys.exit(1)
 
 
+def is_existing_tag(tag: str) -> bool:
+    res = subprocess.run(
+        ["git", "show-ref", "--quiet", "--verify", "refs/tags/" + tag], check=False
+    )
+    return res.returncode == 0
+
+
 def find_previous_version(version: str) -> str:
     major, minor, patchlevel = map(int, version.split("."))
     if major != 4: