gh-gl-sync: Add handling to post status to for merge queue

.github/workflows/custom_docker_builds.yml

@@ -25,7 +25,7 @@ jobs:
       matrix:
         include:
           - docker-image: ./images/gh-gl-sync
-            image-tags: ghcr.io/spack/ci-bridge:0.0.44
+            image-tags: ghcr.io/spack/ci-bridge:0.0.45
           - docker-image: ./images/ci-key-clear
             image-tags: ghcr.io/spack/ci-key-clear:0.0.2
           - docker-image: ./images/gitlab-stuckpods

images/gh-gl-sync/SpackCIBridge.py

@@ -5,6 +5,7 @@
 import base64
 from datetime import datetime, timedelta, timezone
 import dateutil.parser
+import enum
 from github import Github
 import json
 import os
@@ -23,6 +24,15 @@
 sentry_sdk.init(traces_sample_rate=0.1)
 
 
+class MQ(enum.Enum):
+    OFF = enum.auto()
+    SKIP = enum.auto()
+    GITLAB = enum.auto()
+
+    def __str__(self):
+        return f"{self.name}".lower()
+
+
 def _durable_subprocess_run(*args, **kwargs):
     """
     Calls subprocess.run with retries/exponential backoff on failure.
@@ -45,7 +55,7 @@ class SpackCIBridge(object):
 
     def __init__(self, gitlab_repo="", gitlab_host="", gitlab_project="", github_project="",
                  disable_status_post=True, sync_draft_prs=False,
-                 main_branch=None, prereq_checks=[]):
+                 main_branch=None, prereq_checks=[], enable_mq=MQ.OFF):
         self.gitlab_repo = gitlab_repo
         self.github_project = github_project
         github_token = os.environ.get('GITHUB_TOKEN')
@@ -76,6 +86,11 @@ def __init__(self, gitlab_repo="", gitlab_host="", gitlab_project="", github_pro
 
         self.prereq_checks = prereq_checks
 
+        self.enable_mq = enable_mq
+
+        if self.enable_mq is MQ.GITLAB:
+            raise Exception("Gitlab intergration for Merge Queue is not implemented")
+
         dt = datetime.now(timezone.utc) + timedelta(minutes=-60)
         self.time_threshold_brief = urllib.parse.quote_plus(dt.isoformat(timespec="seconds"))
 
@@ -90,6 +105,7 @@ def __init__(self, gitlab_repo="", gitlab_host="", gitlab_project="", github_pro
         self.commit_api_template += "/repository/commits/{0}"
 
         self.cached_commits = {}
+        self.cached_branches = []
 
     @atexit.register
     def cleanup():
@@ -305,8 +321,7 @@ def listify_dict(d):
 
     def list_github_protected_branches(self):
         """ Return a list of protected branch names from GitHub."""
-        branches = self.py_gh_repo.get_branches()
-        print("Rate limit after get_branches(): {}".format(self.py_github.rate_limiting[0]))
+        branches = self.get_gh_branches()
         protected_branches = [br.name for br in branches if br.protected]
         protected_branches = sorted(protected_branches)
         if self.currently_running_sha:
@@ -318,6 +333,18 @@ def list_github_protected_branches(self):
             print("    {0}".format(protected_branch))
         return protected_branches
 
+    def list_github_mq_branches(self):
+        """ Return a list of branch names associated with a merge queue"""
+        def is_mq_branch(branch):
+            return branch.name.startswith("gh-readonly-queue/")
+
+        branches = self.get_gh_branches()
+        mq_branches = [(br.name, br.commit.sha) for br in branches if is_mq_branch(br)]
+        print("MQ Branches:")
+        for branch_name, sha in mq_branches:
+            print("   {0} / {1}".format(branch_name, sha))
+        return mq_branches
+
     def list_github_tags(self):
         """ Return a list of tag names from GitHub."""
         tag_list = self.py_gh_repo.get_tags()
@@ -354,6 +381,14 @@ def get_gitlab_pr_branches(self):
         self.gitlab_pr_output = \
             _durable_subprocess_run(branch_args, stdout=subprocess.PIPE).stdout
 
+    def get_gh_branches(self):
+        if self.cached_branches:
+            return self.cached_branches
+
+        self.cached_branches = self.py_gh_repo.get_branches()
+        print("Rate limit after get_branches(): {}".format(self.py_github.rate_limiting[0]))
+        return self.cached_branches
+
     def gitlab_shallow_fetch(self):
         """Perform a shallow fetch from GitLab"""
         fetch_args = ["git", "fetch", "-q", "--depth=1", "gitlab"]
@@ -530,7 +565,7 @@ def get_pipelines_for_branch(self, branch, time_threshold=None):
 
         return self.dedupe_pipelines(pipelines)
 
-    def post_pipeline_status(self, open_prs, protected_branches):
+    def post_pipeline_status(self, open_prs, protected_branches, skip_branches):
         print("Rate limit at the beginning of post_pipeline_status(): {}".format(self.py_github.rate_limiting[0]))
         pipeline_branches = []
         backlog_branches = []
@@ -595,6 +630,12 @@ def post_pipeline_status(self, open_prs, protected_branches):
         for sha in self.unmergeable_shas:
             print('  {0}'.format(sha))
             self.create_status_for_commit(sha, "", "error", "", f"PR could not be merged with {self.main_branch}")
+
+        # Post a skip status for skip branches
+        for branch_name, sha in skip_branches:
+            print("Posting skipped status for {} / {}".format(branch_name, sha))
+            self.create_status_for_commit(sha, branch_name, "success", "", "Skipped Gitlab CI for branch")
+
         print("Rate limit at the end of post_pipeline_status(): {}".format(self.py_github.rate_limiting[0]))
 
     def create_status_for_commit(self, sha, branch, state, target_url, description):
@@ -664,6 +705,10 @@ def sync(self):
             # Get protected branches on GitHub.
             protected_branches = self.list_github_protected_branches()
 
+            skip_branches = []
+            if self.enable_mq is MQ.SKIP:
+                skip_branches.extend(self.list_github_mq_branches())
+
             # Get tags on GitHub.
             tags = self.list_github_tags()
 
@@ -684,7 +729,7 @@ def sync(self):
             # Post pipeline status to GitHub for each open PR, if enabled
             if self.post_status:
                 print('Posting pipeline status for open PRs and protected branches')
-                self.post_pipeline_status(all_open_prs, protected_branches)
+                self.post_pipeline_status(all_open_prs, protected_branches, skip_branches)
 
 
 if __name__ == "__main__":
@@ -707,6 +752,8 @@ def sync(self):
 on a commit of the main branch that is newer than the latest commit tested by GitLab.""")
     parser.add_argument("--prereq-check", nargs="+", default=False,
                         help="Only push branches that have already passed this GitHub check")
+    parser.add_argument("--enable-mq", default="off", choices=[str(opt) for opt in list(MQ)],
+                        help="Configure how to post statuses for merge queue branches")
 
     args = parser.parse_args()
 
@@ -724,6 +771,7 @@ def sync(self):
                            disable_status_post=args.disable_status_post,
                            sync_draft_prs=args.sync_draft_prs,
                            main_branch=args.main_branch,
-                           prereq_checks=args.prereq_check)
+                           prereq_checks=args.prereq_check,
+                           enable_mq=MQ[args.enable_mq.upper()])
     bridge.setup_ssh(ssh_key_base64)
     bridge.sync()

images/gh-gl-sync/test_SpackCIBridge.py

@@ -445,7 +445,7 @@ def test_post_pipeline_status(capfd):
     bridge.session = session
     os.environ["GITHUB_TOKEN"] = "my_github_token"
 
-    bridge.post_pipeline_status(open_prs, [])
+    bridge.post_pipeline_status(open_prs, [], [])
     assert bridge.session.get.call_count == 2
     assert gh_repo.get_commit.call_count == 1
     assert gh_commit.create_status.call_count == 1
@@ -507,7 +507,7 @@ def verify_backlogged_by_checks(capfd, checks_return_value):
         expected_desc = all_open_prs["backlogged"][0]
         assert expected_desc == "waiting for style check to succeed"
 
-        bridge.post_pipeline_status(all_open_prs, [])
+        bridge.post_pipeline_status(all_open_prs, [], [])
         assert gh_commit.create_status.call_count == 1
         gh_commit.create_status.assert_called_with(
             state="pending",
@@ -561,7 +561,7 @@ def test_pipeline_status_backlogged_for_draft_PR(capfd):
 
     expected_desc = "GitLab CI is disabled for draft PRs"
 
-    bridge.post_pipeline_status(open_prs, [])
+    bridge.post_pipeline_status(open_prs, [], [])
     assert gh_commit.create_status.call_count == 1
     gh_commit.create_status.assert_called_with(
         state="pending",
@@ -574,3 +574,51 @@ def test_pipeline_status_backlogged_for_draft_PR(capfd):
   pr1_readme -> shabaz"""
     assert expected_content in out
     del os.environ["GITHUB_TOKEN"]
+
+
+def test_skipped_status_for_merge_queue(capfd):
+    """Test skipping of merge queue branches."""
+    os.environ["GITHUB_TOKEN"] = "my_github_token"
+
+    github_branches_response = [
+        AttrDict({
+            "name": "gh-readonly-queue/my_topic_branch",
+            "commit": {
+                "sha": "shafoo"
+            },
+            "protected": False
+        }),
+    ]
+    gh_repo = Mock()
+    gh_repo.get_branches.return_value = github_branches_response
+
+    gh_commit = Mock()
+    gh_commit.get_combined_status.return_value = AttrDict({'statuses': []})
+    gh_commit.create_status.return_value = AttrDict({"state": "success"})
+    gh_repo.get_commit.return_value = gh_commit
+
+    bridge = SpackCIBridge.SpackCIBridge(enable_mq=SpackCIBridge.MQ['SKIP'])
+    bridge.py_gh_repo = gh_repo
+    bridge.py_github = py_github
+    skip_branches = bridge.list_github_mq_branches()
+
+    open_prs = {
+        "pr_strings": [],
+        "base_shas": [],
+        "head_shas": [],
+        "backlogged": []
+    }
+
+    expected_desc = "Skipped Gitlab CI for branch"
+    bridge.post_pipeline_status(open_prs, [], skip_branches)
+    assert gh_commit.create_status.call_count == 1
+    gh_commit.create_status.assert_called_with(
+        state="success",
+        context="ci/gitlab-ci",
+        description=expected_desc,
+        target_url=""
+    )
+    out, err = capfd.readouterr()
+    expected_content = "Posting skipped status for gh-readonly-queue/my_topic_branch"
+    assert expected_content in out
+    del os.environ["GITHUB_TOKEN"]

k8s/production/custom/gh-gl-sync/cron-jobs.yaml

@@ -16,7 +16,7 @@ spec:
           restartPolicy: Never
           containers:
           - name: sync
-            image: ghcr.io/spack/ci-bridge:0.0.44
+            image: ghcr.io/spack/ci-bridge:0.0.45
             imagePullPolicy: IfNotPresent
             resources:
               requests:

k8s/production/custom/kokkos-sync/cron-jobs.yaml

@@ -16,7 +16,7 @@ spec:
           restartPolicy: Never
           containers:
           - name: sync
-            image: ghcr.io/spack/ci-bridge:0.0.44
+            image: ghcr.io/spack/ci-bridge:0.0.45
             imagePullPolicy: IfNotPresent
             resources:
               requests: