emerge-gitclone: Rework to simply clone scripts + submodules into ~core

emerge-gitclone

@@ -2,17 +2,20 @@
 # Copyright The Flatcar Authors.
 # SPDX-License-Identifier: Apache-2.0
 
+import argparse
 import os
+import re
 import shutil
 import subprocess
 import sys
-import re
+import urllib.parse
+import urllib.request
 
 import portage
 
 
-GIT_URI = "https://github.com/flatcar/{repo}.git"
-GIT_LOCATION = "/var/lib/portage/{repo}"
+GH_REPO = "flatcar/scripts"
+GIT_LOCATION = "/home/core/scripts"
 
 
 def get_release():
@@ -39,91 +42,97 @@ def get_channel():
     return channel
 
 
-def git_clone_scripts(repo_url, repo_location, ref):
+def git_clone_scripts(ref):
     """Clone the scripts repo at the given location, and handle submodules (if applicable)"""
-    print(f">>> Starting git clone in {repo_location}")
+    print(f">>> Starting git clone at {ref} in {GIT_LOCATION}")
     os.umask(0o022)
-    subprocess.check_call(
-        ["git", "clone", repo_url, repo_location]
+    subprocess.check_call([
+        "git",
+        "clone",
+        "--depth=1",
+        "--revision", ref,
+        f"https://github.com/{GH_REPO}.git", GIT_LOCATION,
+    ])
+    # We could handle the submodules with the clone above, but the
+    # --single-branch option does not propagate.
+    subprocess.check_call([
+        "git", "-C", GIT_LOCATION,
+        "submodule", "update",
+        "--init",
+        "--recursive",
+        "--depth=1",
+        "--single-branch",
+    ])
+    print(f">>> Git clone in {GIT_LOCATION} successful")
+
+
+def resolve_revision(revision):
+    """Resolve a shortened commit SHA to the full SHA through GitHub."""
+    if not re.fullmatch(r"[0-9A-Fa-f]{4,39}", revision):
+        return revision
+
+    encoded_revision = urllib.parse.quote(revision, safe="")
+    request = urllib.request.Request(
+        f"https://api.github.com/repos/{GH_REPO}/commits/{encoded_revision}",
+        headers={"Accept": "application/vnd.github.sha"},
     )
+    with urllib.request.urlopen(request) as response:
+        resolved_revision = response.read().decode("ascii").strip()
 
-    # Check if this ref uses submodules and check these out if it does.
-    subprocess.check_output(
-        ["git", "-C", repo_location, "checkout", ref]
-    )
-    try:
-        with open(repo_location + '/.gitmodules') as gm:
-            gmc = gm.read()
-            if '[submodule ' in gmc:
-                print(f">>> Submodules detected in {ref}.")
-                subprocess.check_output(
-                    ["git", "-C", repo_location, "submodule", "init"]
-                )
-                subprocess.check_output(
-                    ["git", "-C", repo_location, "submodule", "update"]
-                )
-            else:
-                print(f">>> Empty or invalid submodule config detected in {ref}")
-                print(f">>>  at {repo_location}/.gitmodules. Ignoring and continuing w/o submodules.")
-    except (FileNotFoundError, IOError):
-        print(f">>> No submodules detected in {ref}. Assuming unified scripts repo.")
-
-    print(f">>> Git clone in {repo_location} successful")
-
-
-def sync_repo():
-    release = get_release()
-    channel = get_channel()
-    ref = f"{channel}-{release}"
-
-    # Check if release ends with a git ref. If it does we're dealing with
-    #   a dev build which does not necessarily have a tag on 'scripts'.
-    # These version strings look like e.g. "3552.0.0+nightly-20230323-2100-4-g5d72f4ee",
-    #   with "-g<checksum>" being a git ref.
-    gitref_pat = r'-g([0-9A-Fa-f]+)$'
-    ref_match = re.search(gitref_pat, release)
-    if ref_match:
-        git_ref = ref_match.group(1)
-        print(f">>> Gitref detected in version {ref}, will check out {git_ref}.")
-        ref = git_ref
-
-    repo_list = ["scripts", "coreos-overlay", "portage-stable"]
-
-    for repo in repo_list:
-        if os.path.isdir(GIT_LOCATION.format(repo=repo)) and not os.path.islink(
-            GIT_LOCATION.format(repo=repo)
-        ):
-            shutil.rmtree(GIT_LOCATION.format(repo=repo))
-        if os.path.lexists(GIT_LOCATION.format(repo=repo)):
-            os.unlink(GIT_LOCATION.format(repo=repo))
-
-        if repo == "scripts":
-            git_clone_scripts(
-                repo_url=GIT_URI.format(repo=repo),
-                repo_location=GIT_LOCATION.format(repo=repo),
-                ref=ref
-            )
-
-        else:
-            print(f">>> Symlink the repository with the appropriate folder - {repo}")
-            os.symlink(
-                GIT_LOCATION.format(repo="scripts")
-                + f"/sdk_container/src/third_party/{repo}",
-                GIT_LOCATION.format(repo=repo),
-            )
+    if not re.fullmatch(r"[0-9A-Fa-f]{40}", resolved_revision):
+        raise ValueError(f"GitHub returned an invalid SHA for revision {revision}")
+
+    print(f">>> Resolved revision {revision} to {resolved_revision}.")
+    return resolved_revision
+
+
+def sync_repo(revision=None):
+    if revision is not None:
+        ref = revision
+    else:
+        release = get_release()
+        channel = get_channel()
+        ref = f"{channel}-{release}"
+
+        # Check if release ends with a git ref. If it does we're dealing with
+        #   a dev build which does not necessarily have a tag on 'scripts'.
+        # These version strings look like e.g. "3552.0.0+nightly-20230323-2100-4-g5d72f4ee",
+        #   with "-g<checksum>" being a git ref.
+        gitref_pat = r'-g([0-9A-Fa-f]+)$'
+        ref_match = re.search(gitref_pat, release)
+        if ref_match:
+            git_ref = ref_match.group(1)
+            print(f">>> Gitref detected in version {ref}, will check out {git_ref}.")
+            ref = git_ref
+
+    ref = resolve_revision(ref)
+
+    if os.path.isdir(GIT_LOCATION) and not os.path.islink(GIT_LOCATION):
+        shutil.rmtree(GIT_LOCATION)
+    elif os.path.lexists(GIT_LOCATION):
+        os.unlink(GIT_LOCATION)
+
+    git_clone_scripts(ref=ref)
 
 
 def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--revision",
+        help="scripts repository revision to clone instead of the release-derived ref",
+    )
+    args = parser.parse_args()
+
     try:
-        sync_repo()
+        sync_repo(revision=args.revision)
         # Perform normal post-sync tasks
         configroot = portage.settings["PORTAGE_CONFIGROOT"]
         post_sync = "%s/etc/portage/bin/post_sync" % configroot
         if os.path.exists(post_sync):
             subprocess.check_call([post_sync])
         subprocess.check_call(["emerge", "--check-news", "--quiet"])
     except Exception as e:
-        print(e.output)
+        print(e)
         sys.stderr.write(">>> No git repositories configured.\n")
         sys.exit(1)