feat(ci_visibility): use more efficient git unshallow method if available

ddtrace/ext/git.py

@@ -197,20 +197,51 @@ def _get_device_for_path(path):
     return os.stat(path).st_dev
 
 
+@cached()
+def _get_remote_name(cwd=None):
+    # type: (Optional[str]) -> Optional[str]
+    remote, stderr, _, exit_code = _extract_clone_defaultremotename_with_details(cwd=cwd)
+    if exit_code != 0:
+        log.debug("Failed to get default remote: %s", stderr)
+        return None
+
+    return remote
+
+
 def _unshallow_repository_with_details(
     cwd: Optional[str] = None, repo: Optional[str] = None, refspec: Optional[str] = None, parent_only: bool = False
 ) -> _GitSubprocessDetails:
-    cmd = [
-        "fetch",
-        "--deepen=1" if parent_only else '--shallow-since="1 month ago"',
-        "--update-shallow",
-        "--filter=blob:none",
-        "--recurse-submodules=no",
-    ]
-    if repo is not None:
-        cmd.append(repo)
-    if refspec is not None:
-        cmd.append(refspec)
+    if repo and refspec and extract_git_version(cwd) >= (2, 36, 0):
+        log.debug("Shallow repository on git >= 2.36 detected, unshallowing with new algorithm")
+        cmd = [
+            "-c",
+            "gc.auto=0",
+            "-c",
+            "fetch.negotiationAlgorithm=skipping",
+            "-c",
+            f"remote.{repo}.promisor=true",
+            "fetch",
+            '--shallow-since="1 month ago"',
+            "--update-shallow",
+            "--filter=tree:0",
+            "--no-tags",
+            "--recurse-submodules=no",
+            "--no-write-fetch-head",
+            repo,
+            refspec,
+        ]
+    else:
+        cmd = [
+            "fetch",
+            "--deepen=1" if parent_only else '--shallow-since="1 month ago"',
+            "--update-shallow",
+            "--filter=blob:none",
+            "--recurse-submodules=no",
+        ]
+        if repo is not None:
+            cmd.append(repo)
+        if refspec is not None:
+            cmd.append(refspec)
 
     return _git_subprocess_cmd_with_details(*cmd, cwd=cwd)
 
@@ -224,6 +255,36 @@ def _unshallow_repository(
     _unshallow_repository_with_details(cwd, repo, refspec, parent_only)
 
 
+def _ensure_fetch_unshallowed_trees(refspecs: List[str], cwd: Optional[str] = None) -> None:
+    if extract_git_version(cwd=cwd) < (2, 36, 0):
+        return
+
+    remote = _get_remote_name(cwd=cwd)
+    if not remote:
+        return
+
+    cmd = [
+        "git",
+        "-c",
+        "gc.auto=0",
+        "-c",
+        "fetch.negotiationAlgorithm=skipping",
+        "-c",
+        f"remote.{repo}.promisor=true",
+        "fetch",
+        "--refetch",
+        '--shallow-since="1 month ago"',
+        "--update-shallow",
+        "--filter=blob:none",
+        "--no-tags",
+        "--recurse-submodules=no",
+        repo,
+        "HEAD",
+        *refspecs,
+    ]
+    _git_subprocess_cmd_with_details(*cmd, cwd=cwd)
+
+
 def extract_user_info(cwd: Optional[str] = None, commit_sha: Optional[str] = None) -> Dict[str, Tuple[str, str, str]]:
     """Extract commit author info from the git repository in the current directory or one specified by ``cwd``."""
     # Note: `git show -s --format... --date...` is supported since git 2.1.4 onwards
@@ -238,6 +299,7 @@ def extract_user_info(cwd: Optional[str] = None, commit_sha: Optional[str] = Non
     }
 
 
+@cached()
 def extract_git_version(cwd=None):
     output = _git_subprocess_cmd("--version")
     try:
@@ -350,7 +412,8 @@ def extract_git_head_metadata(head_commit_sha: str, cwd: Optional[str] = None) -
 
     is_shallow, *_ = _is_shallow_repository_with_details(cwd=cwd)
     if is_shallow:
-        _unshallow_repository(cwd=cwd, repo=None, refspec=None, parent_only=True)
+        remote = _get_remote_name(cwd=cwd)
+        _unshallow_repository(cwd=cwd, repo=remote, refspec=head_commit_sha)
 
     try:
         users = extract_user_info(cwd=cwd, commit_sha=head_commit_sha)

ddtrace/internal/ci_visibility/git_client.py

@@ -13,6 +13,7 @@
 
 from ddtrace.ext import ci
 from ddtrace.ext.git import _build_git_packfiles_with_details
+from ddtrace.ext.git import _ensure_fetch_unshallowed_trees
 from ddtrace.ext.git import _extract_clone_defaultremotename_with_details
 from ddtrace.ext.git import _extract_latest_commits_with_details
 from ddtrace.ext.git import _extract_upstream_sha
@@ -177,6 +178,7 @@ def _run_protocol(
         log_level=0,  # int
     ):
         # type: (...) -> None
+        unshallow_performed = False
         log.setLevel(log_level)
         _metadata_upload_status.value = METADATA_UPLOAD_STATUS.IN_PROCESS
         try:
@@ -206,24 +208,30 @@ def _run_protocol(
                 log.debug("Shallow repository detected on git > 2.27 detected, unshallowing")
                 try:
                     cls._unshallow_repository(cwd=cwd)
+                    unshallow_performed = True
                 except ValueError:
                     log.warning("Failed to unshallow repository, continuing to send pack data", exc_info=True)
 
-            latest_commits = cls._get_latest_commits(cwd=cwd)
-            backend_commits = cls._search_commits(
-                requests_mode, base_url, repo_url, latest_commits, serializer, _response
-            )
-            if backend_commits is None:
-                log.debug("No backend commits found, returning early.")
-                _metadata_upload_status.value = METADATA_UPLOAD_STATUS.FAILED
-                return
+            if unshallow_performed:
+                # DEV: we only need to fetch data from backend again if unshallow was performed, otherwise it will be
+                # the same as before.
+                latest_commits = cls._get_latest_commits(cwd=cwd)
+                backend_commits = cls._search_commits(
+                    requests_mode, base_url, repo_url, latest_commits, serializer, _response
+                )
+                if backend_commits is None:
+                    log.debug("No backend commits found, returning early.")
+                    _metadata_upload_status.value = METADATA_UPLOAD_STATUS.FAILED
+                    return
 
-            commits_not_in_backend = list(set(latest_commits) - set(backend_commits))
+                commits_not_in_backend = list(set(latest_commits) - set(backend_commits))
 
             rev_list = cls._get_filtered_revisions(
                 excluded_commits=backend_commits, included_commits=commits_not_in_backend, cwd=cwd
             )
             if rev_list:
+                if unshallow_performed:
+                    cls._ensure_fetch_unshallowed_trees(cwd=cwd, refspecs=rev_list)
                 log.debug("Building and uploading packfiles for revision list: %s", rev_list)
                 with _build_git_packfiles_with_details(rev_list, cwd=cwd) as (packfiles_prefix, packfiles_details):
                     record_git_command(
@@ -433,19 +441,25 @@ def _unshallow_repository(cls, cwd=None):
 
     @classmethod
     def _unshallow_repository_to_local_head(cls, remote, cwd=None):
-        # type (str, Optional[str) -> None
+        # type (str, Optional[str]) -> None
         head = extract_commit_sha(cwd=cwd)
         log.debug("Unshallowing to local head %s", head)
         _unshallow_repository(cwd=cwd, repo=remote, refspec=head)
         log.debug("Unshallowing to local head successful")
 
     @classmethod
     def _unshallow_repository_to_upstream(cls, remote, cwd=None):
-        # type (str, Optional[str) -> None
+        # type (str, Optional[str]) -> None
         upstream = _extract_upstream_sha(cwd=cwd)
         log.debug("Unshallowing to upstream %s", upstream)
         _unshallow_repository(cwd=cwd, repo=remote, refspec=upstream)
-        log.debug("Unshallowing to upstream")
+        log.debug("Unshallowing to upstream successful")
+
+    @classmethod
+    def _ensure_fetch_unshallowed_trees(cls, refspecs, cwd=None):
+        log.debug("Ensuring unshallowed trees fetched")
+        _ensure_fetch_unshallowed_trees(cwd=cwd, refspecs=refspecs)
+        log.debug("Unshallowed trees fetched")
 
 
 class CIVisibilityGitClientSerializerV1(object):