From 413854f187f33ceaf572b9df7b6e6e4928977d0d Mon Sep 17 00:00:00 2001 From: James Le Cuirot Date: Thu, 17 Sep 2026 14:51:53 +0100 Subject: [PATCH 1/3] emerge-gitclone: Rework to simply clone scripts + submodules into ~core The portage-stable and coreos-overlay repositories got absorbed into scripts a long time ago. Now portage-stable is being replaced by a gentoo repo submodule. This script now simply does a shallow single branch clone of scripts and any submodules. This will succeed, even if there are no submodules. Storing scripts under /var/lib/portage was weird, so it is now cloned under ~core instead, with the Portage configuration pointing at the repositories within this location. It is now expected that this script is called as the core user rather than root. Signed-off-by: James Le Cuirot --- emerge-gitclone | 82 +++++++++++++++++-------------------------------- 1 file changed, 28 insertions(+), 54 deletions(-) diff --git a/emerge-gitclone b/emerge-gitclone index 8908eb5..dba8cf6 100755 --- a/emerge-gitclone +++ b/emerge-gitclone @@ -11,8 +11,8 @@ import re import portage -GIT_URI = "https://github.com/flatcar/{repo}.git" -GIT_LOCATION = "/var/lib/portage/{repo}" +GH_REPO = "flatcar/scripts" +GIT_LOCATION = "/home/core/scripts" def get_release(): @@ -39,36 +39,28 @@ def get_channel(): return channel -def git_clone_scripts(repo_url, repo_location, ref): +def git_clone_scripts(ref): """Clone the scripts repo at the given location, and handle submodules (if applicable)""" - print(f">>> Starting git clone in {repo_location}") + print(f">>> Starting git clone at {ref} in {GIT_LOCATION}") os.umask(0o022) - subprocess.check_call( - ["git", "clone", repo_url, repo_location] - ) - - # Check if this ref uses submodules and check these out if it does. - subprocess.check_output( - ["git", "-C", repo_location, "checkout", ref] - ) - try: - with open(repo_location + '/.gitmodules') as gm: - gmc = gm.read() - if '[submodule ' in gmc: - print(f">>> Submodules detected in {ref}.") - subprocess.check_output( - ["git", "-C", repo_location, "submodule", "init"] - ) - subprocess.check_output( - ["git", "-C", repo_location, "submodule", "update"] - ) - else: - print(f">>> Empty or invalid submodule config detected in {ref}") - print(f">>> at {repo_location}/.gitmodules. Ignoring and continuing w/o submodules.") - except (FileNotFoundError, IOError): - print(f">>> No submodules detected in {ref}. Assuming unified scripts repo.") - - print(f">>> Git clone in {repo_location} successful") + subprocess.check_call([ + "git", + "clone", + "--depth=1", + "--revision", ref, + f"https://github.com/{GH_REPO}.git", GIT_LOCATION, + ]) + # We could handle the submodules with the clone above, but the + # --single-branch option does not propagate. + subprocess.check_call([ + "git", "-C", GIT_LOCATION, + "submodule", "update", + "--init", + "--recursive", + "--depth=1", + "--single-branch", + ]) + print(f">>> Git clone in {GIT_LOCATION} successful") def sync_repo(): @@ -87,30 +79,12 @@ def sync_repo(): print(f">>> Gitref detected in version {ref}, will check out {git_ref}.") ref = git_ref - repo_list = ["scripts", "coreos-overlay", "portage-stable"] - - for repo in repo_list: - if os.path.isdir(GIT_LOCATION.format(repo=repo)) and not os.path.islink( - GIT_LOCATION.format(repo=repo) - ): - shutil.rmtree(GIT_LOCATION.format(repo=repo)) - if os.path.lexists(GIT_LOCATION.format(repo=repo)): - os.unlink(GIT_LOCATION.format(repo=repo)) - - if repo == "scripts": - git_clone_scripts( - repo_url=GIT_URI.format(repo=repo), - repo_location=GIT_LOCATION.format(repo=repo), - ref=ref - ) - - else: - print(f">>> Symlink the repository with the appropriate folder - {repo}") - os.symlink( - GIT_LOCATION.format(repo="scripts") - + f"/sdk_container/src/third_party/{repo}", - GIT_LOCATION.format(repo=repo), - ) + if os.path.isdir(GIT_LOCATION) and not os.path.islink(GIT_LOCATION): + shutil.rmtree(GIT_LOCATION) + elif os.path.lexists(GIT_LOCATION): + os.unlink(GIT_LOCATION) + + git_clone_scripts(ref=ref) def main(): From 60b3c39629db56a47ce5600201d7bd4125887f97 Mon Sep 17 00:00:00 2001 From: James Le Cuirot Date: Thu, 17 Sep 2026 14:57:44 +0100 Subject: [PATCH 2/3] emerge-gitclone: Add an option to override the git revision This is useful for testing. Signed-off-by: James Le Cuirot --- emerge-gitclone | 43 +++++++++++++++++++++++++++---------------- 1 file changed, 27 insertions(+), 16 deletions(-) diff --git a/emerge-gitclone b/emerge-gitclone index dba8cf6..588d041 100755 --- a/emerge-gitclone +++ b/emerge-gitclone @@ -2,6 +2,7 @@ # Copyright The Flatcar Authors. # SPDX-License-Identifier: Apache-2.0 +import argparse import os import shutil import subprocess @@ -63,21 +64,24 @@ def git_clone_scripts(ref): print(f">>> Git clone in {GIT_LOCATION} successful") -def sync_repo(): - release = get_release() - channel = get_channel() - ref = f"{channel}-{release}" - - # Check if release ends with a git ref. If it does we're dealing with - # a dev build which does not necessarily have a tag on 'scripts'. - # These version strings look like e.g. "3552.0.0+nightly-20230323-2100-4-g5d72f4ee", - # with "-g" being a git ref. - gitref_pat = r'-g([0-9A-Fa-f]+)$' - ref_match = re.search(gitref_pat, release) - if ref_match: - git_ref = ref_match.group(1) - print(f">>> Gitref detected in version {ref}, will check out {git_ref}.") - ref = git_ref +def sync_repo(revision=None): + if revision is not None: + ref = revision + else: + release = get_release() + channel = get_channel() + ref = f"{channel}-{release}" + + # Check if release ends with a git ref. If it does we're dealing with + # a dev build which does not necessarily have a tag on 'scripts'. + # These version strings look like e.g. "3552.0.0+nightly-20230323-2100-4-g5d72f4ee", + # with "-g" being a git ref. + gitref_pat = r'-g([0-9A-Fa-f]+)$' + ref_match = re.search(gitref_pat, release) + if ref_match: + git_ref = ref_match.group(1) + print(f">>> Gitref detected in version {ref}, will check out {git_ref}.") + ref = git_ref if os.path.isdir(GIT_LOCATION) and not os.path.islink(GIT_LOCATION): shutil.rmtree(GIT_LOCATION) @@ -88,8 +92,15 @@ def sync_repo(): def main(): + parser = argparse.ArgumentParser() + parser.add_argument( + "--revision", + help="scripts repository revision to clone instead of the release-derived ref", + ) + args = parser.parse_args() + try: - sync_repo() + sync_repo(revision=args.revision) # Perform normal post-sync tasks configroot = portage.settings["PORTAGE_CONFIGROOT"] post_sync = "%s/etc/portage/bin/post_sync" % configroot From 4ac45aac30a22e0f47d69e156a9c0a9a7d5cbd57 Mon Sep 17 00:00:00 2001 From: James Le Cuirot Date: Thu, 17 Sep 2026 15:24:22 +0100 Subject: [PATCH 3/3] emerge-gitclone: Fix cloning with a shortened git SHA git clone --revision requires a full SHA, so if a Flatcar dev build has a -g suffix, we need to resolve the SHA using the GitHub API first. Signed-off-by: James Le Cuirot --- emerge-gitclone | 28 ++++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/emerge-gitclone b/emerge-gitclone index 588d041..476f968 100755 --- a/emerge-gitclone +++ b/emerge-gitclone @@ -4,10 +4,12 @@ import argparse import os +import re import shutil import subprocess import sys -import re +import urllib.parse +import urllib.request import portage @@ -64,6 +66,26 @@ def git_clone_scripts(ref): print(f">>> Git clone in {GIT_LOCATION} successful") +def resolve_revision(revision): + """Resolve a shortened commit SHA to the full SHA through GitHub.""" + if not re.fullmatch(r"[0-9A-Fa-f]{4,39}", revision): + return revision + + encoded_revision = urllib.parse.quote(revision, safe="") + request = urllib.request.Request( + f"https://api.github.com/repos/{GH_REPO}/commits/{encoded_revision}", + headers={"Accept": "application/vnd.github.sha"}, + ) + with urllib.request.urlopen(request) as response: + resolved_revision = response.read().decode("ascii").strip() + + if not re.fullmatch(r"[0-9A-Fa-f]{40}", resolved_revision): + raise ValueError(f"GitHub returned an invalid SHA for revision {revision}") + + print(f">>> Resolved revision {revision} to {resolved_revision}.") + return resolved_revision + + def sync_repo(revision=None): if revision is not None: ref = revision @@ -83,6 +105,8 @@ def sync_repo(revision=None): print(f">>> Gitref detected in version {ref}, will check out {git_ref}.") ref = git_ref + ref = resolve_revision(ref) + if os.path.isdir(GIT_LOCATION) and not os.path.islink(GIT_LOCATION): shutil.rmtree(GIT_LOCATION) elif os.path.lexists(GIT_LOCATION): @@ -108,7 +132,7 @@ def main(): subprocess.check_call([post_sync]) subprocess.check_call(["emerge", "--check-news", "--quiet"]) except Exception as e: - print(e.output) + print(e) sys.stderr.write(">>> No git repositories configured.\n") sys.exit(1)