pull secret propagation and registry authentication - milestone 5b

.github/workflows/ci.yaml

@@ -8,7 +8,7 @@ on:
     branches: [main]
 
 env:
-  BINK_COMMIT: a38101a436b96a82698363f0d07d8dba2cf088c6
+  BINK_COMMIT: e7d4574fdc610a9fd9bd067aff984516ec7c507d
 
 permissions: {}
 

Makefile

@@ -9,6 +9,8 @@ ARTIFACTS ?= $(abspath _output/logs)
 DEFAULT_KUBE_MINOR ?= 1.35
 BINK_NODE_DISK_IMAGE ?= ghcr.io/bootc-dev/bink/node:v$(DEFAULT_KUBE_MINOR)-fedora-44-disk
 BINK_LOCAL_REGISTRY_NODE_IMAGE ?= registry.cluster.local:5000/node
+E2E_REGISTRY_USER ?= e2e-user
+E2E_REGISTRY_PASSWORD ?= e2e-password
 # YEAR defines the year value used for substituting the YEAR placeholder in the boilerplate header.
 YEAR ?= $(shell date +%Y)
 
@@ -96,6 +98,7 @@ e2e: ## Run e2e tests (requires: make deploy-bink). V=1 for verbose. RUN=<regex>
 		BINK_NODE_IMAGE_DIGEST=$$(skopeo inspect --tls-verify=false --format '{{.Digest}}' docker://localhost:5000/node:latest) \
 		BINK_NODE_IMAGE_UPDATE_DIGEST=$$(skopeo inspect --tls-verify=false docker://localhost:5000/node:update | jq -r '.Digest') \
 		BINK_NODE_IMAGE_UPDATE2_DIGEST=$$(skopeo inspect --tls-verify=false docker://localhost:5000/node:update2 | jq -r '.Digest') \
+		E2E_REGISTRY_USER=$(E2E_REGISTRY_USER) E2E_REGISTRY_PASSWORD=$(E2E_REGISTRY_PASSWORD) \
 		go test -timeout 30m -count=1 $(if $(V),-v) $(if $(RUN),-run $(RUN)) .
 
 ##@ Build
@@ -169,7 +172,8 @@ start-bink: seed-node-image ## Start a bink cluster (idempotent).
 	bink cluster list 2>&1 | grep -qw $(BINK_CLUSTER_NAME) || { \
 		node_digest=$$(skopeo inspect --tls-verify=false --format '{{.Digest}}' docker://localhost:5000/node:latest) && \
 		bink cluster start --cluster-name $(BINK_CLUSTER_NAME) --node-name controller --api-port 0 --expose $(KUBECONFIG_BINK) \
-		--node-image $(BINK_NODE_DISK_IMAGE) --target-imgref $(BINK_LOCAL_REGISTRY_NODE_IMAGE)@$$node_digest; }
+		--node-image $(BINK_NODE_DISK_IMAGE) --target-imgref $(BINK_LOCAL_REGISTRY_NODE_IMAGE)@$$node_digest \
+		--registry-user $(E2E_REGISTRY_USER) --registry-password $(E2E_REGISTRY_PASSWORD); }
 	kubectl --kubeconfig $(KUBECONFIG_BINK) wait --for=condition=Ready node/controller --timeout=5m
 
 .PHONY: deploy-bink

api/v1alpha1/bootcnode_types.go

@@ -118,12 +118,6 @@ type BootcNodeSpec struct {
 	// credentials. Copied from the owning pool's spec.
 	// +optional
 	PullSecretRef *PullSecretRef `json:"pullSecretRef,omitempty"`
-
-	// pullSecretHash is a hash of the pull secret's content, used to
-	// detect changes. When this value changes, the daemon re-fetches
-	// the secret and updates the host filesystem.
-	// +optional
-	PullSecretHash string `json:"pullSecretHash,omitempty"`
 }
 
 // BootcNodeStatus defines the observed state of a BootcNode.

api/v1alpha1/bootcnodepool_types.go

@@ -56,6 +56,10 @@ const (
 	// resolve the tag.
 	PoolTagResolutionError string = "TagResolutionError"
 
+	// PoolSecretError means the referenced pull secret is missing or
+	// does not contain the expected key.
+	PoolSecretError string = "SecretError"
+
 	// PoolHealthy means no issues.
 	PoolHealthy string = "Healthy"
 )

cmd/daemon/main.go

@@ -94,6 +94,7 @@ func main() {
 		Client:        mgr.GetClient(),
 		Scheme:        mgr.GetScheme(),
 		NodeName:      nodeName,
+		HostRoot:      "/proc/1/root",
 		Executor:      executor,
 		StatusWatcher: watcher,
 	}).SetupWithManager(mgr); err != nil {

config/crd/bases/node.bootc.dev_bootcnodes.yaml

@@ -62,12 +62,6 @@ spec:
                 - Staged
                 - Booted
                 type: string
-              pullSecretHash:
-                description: |-
-                  pullSecretHash is a hash of the pull secret's content, used to
-                  detect changes. When this value changes, the daemon re-fetches
-                  the secret and updates the host filesystem.
-                type: string
               pullSecretRef:
                 description: |-
                   pullSecretRef references a Secret containing image pull

config/rbac/daemon_role.yaml

@@ -6,6 +6,12 @@ metadata:
     app.kubernetes.io/managed-by: kustomize
   name: daemon-role
 rules:
+- apiGroups:
+  - ""
+  resources:
+  - secrets
+  verbs:
+  - get
 - apiGroups:
   - node.bootc.dev
   resources:

config/rbac/role.yaml

@@ -34,6 +34,14 @@ rules:
   - pods/eviction
   verbs:
   - create
+- apiGroups:
+  - ""
+  resources:
+  - secrets
+  verbs:
+  - get
+  - list
+  - watch
 - apiGroups:
   - apps
   resources:

internal/controller/bootcnodepool_controller.go

@@ -48,7 +48,7 @@ type drainStatus struct {
 
 // TagResolver resolves a container image reference to a digest.
 type TagResolver interface {
-	Resolve(ctx context.Context, ref string) (string, error)
+	Resolve(ctx context.Context, ref string, auth []byte) (string, error)
 }
 
 // BootcNodePoolReconciler reconciles a BootcNodePool object
@@ -79,6 +79,7 @@ type BootcNodePoolReconciler struct {
 // +kubebuilder:rbac:groups=node.bootc.dev,resources=bootcnodes,verbs=get;list;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups=node.bootc.dev,resources=bootcnodes/status,verbs=get
 // +kubebuilder:rbac:groups="",resources=nodes,verbs=get;list;watch;update;patch
+// +kubebuilder:rbac:groups="",resources=secrets,verbs=get;list;watch
 // +kubebuilder:rbac:groups="",resources=pods,verbs=get;list
 // +kubebuilder:rbac:groups="",resources=pods/eviction,verbs=create
 // +kubebuilder:rbac:groups=apps,resources=daemonsets,verbs=get
@@ -94,6 +95,7 @@ func (r *BootcNodePoolReconciler) SetupWithManager(mgr ctrl.Manager) error {
 		For(&bootcv1alpha1.BootcNodePool{}).
 		Owns(&bootcv1alpha1.BootcNode{}).
 		Watches(&corev1.Node{}, handler.EnqueueRequestsFromMapFunc(r.mapNodeToPoolRequests), builder.WithPredicates(nodePredicates())).
+		Watches(&corev1.Secret{}, handler.EnqueueRequestsFromMapFunc(r.mapSecretToPoolRequests), builder.WithPredicates(secretTypePredicate())).
 		WatchesRawSource(source.Channel(r.drainCh, &handler.EnqueueRequestForObject{})).
 		Named("bootcnodepool").
 		Complete(r)
@@ -161,6 +163,39 @@ func (r *BootcNodePoolReconciler) mapNodeToPoolRequests(
 	return requests
 }
 
+// mapSecretToPoolRequests maps a Secret event to the BootcNodePool(s) that
+// should be reconciled. It enqueues pools whose pullSecretRef matches the
+// changed Secret's name and namespace.
+func (r *BootcNodePoolReconciler) mapSecretToPoolRequests(
+	ctx context.Context,
+	obj client.Object,
+) []reconcile.Request {
+	secret, ok := obj.(*corev1.Secret)
+	if !ok {
+		return nil
+	}
+
+	var pools bootcv1alpha1.BootcNodePoolList
+	if err := r.List(ctx, &pools); err != nil {
+		log := logf.FromContext(ctx)
+		log.Error(err, "Failed to list BootcNodePools in secret mapper")
+		return nil
+	}
+
+	var requests []reconcile.Request
+	for i := range pools.Items {
+		pool := &pools.Items[i]
+		if pool.Spec.PullSecretRef != nil &&
+			pool.Spec.PullSecretRef.Name == secret.Name &&
+			pool.Spec.PullSecretRef.Namespace == secret.Namespace {
+			requests = append(requests, reconcile.Request{
+				NamespacedName: types.NamespacedName{Name: pool.Name},
+			})
+		}
+	}
+	return requests
+}
+
 // nodeSelectorMatchesNode evaluates whether a node's labels match a
 // LabelSelector.
 func nodeSelectorMatchesNode(sel *metav1.LabelSelector, node *corev1.Node) (bool, error) {
@@ -219,6 +254,18 @@ func nodeUnschedulableChanged(oldNode, newNode *corev1.Node) bool {
 	return oldNode.Spec.Unschedulable != newNode.Spec.Unschedulable
 }
 
+// secretTypePredicate returns a predicate that accepts only Secrets of
+// type kubernetes.io/dockerconfigjson.
+func secretTypePredicate() predicate.Predicate {
+	return predicate.NewPredicateFuncs(func(obj client.Object) bool {
+		secret, ok := obj.(*corev1.Secret)
+		if !ok {
+			return false
+		}
+		return secret.Type == corev1.SecretTypeDockerConfigJson
+	})
+}
+
 // Reconcile is part of the main kubernetes reconciliation loop which aims to
 // move the current state of the cluster closer to the desired state.
 func (r *BootcNodePoolReconciler) Reconcile(
@@ -269,8 +316,12 @@ func (r *BootcNodePoolReconciler) Reconcile(
 	// Status writes to `pool` are also permitted; we Update() back once at
 	// the end.
 
-	// Resolve the target digest from the image ref.
-	resolveResult, err := r.resolveTargetDigest(ctx, &pool)
+	secretData, err := r.fetchPullSecretData(ctx, &pool)
+	if err != nil {
+		log.Error(err, "Failed to fetch pull secret")
+		setPoolDegraded(&pool, bootcv1alpha1.PoolSecretError, err.Error())
+	}
+	resolveResult, err := r.resolveTargetDigest(ctx, &pool, secretData)
 	if err != nil {
 		if isInvalidSpecError(err) {
 			return r.setInvalidSpecCondition(ctx, &pool, err)
@@ -387,6 +438,7 @@ func (r *BootcNodePoolReconciler) handlePoolDeletion(
 func (r *BootcNodePoolReconciler) resolveTargetDigest(
 	ctx context.Context,
 	pool *bootcv1alpha1.BootcNodePool,
+	secretData []byte,
 ) (ctrl.Result, error) {
 	log := logf.FromContext(ctx)
 
@@ -415,7 +467,7 @@ func (r *BootcNodePoolReconciler) resolveTargetDigest(
 		return ctrl.Result{RequeueAfter: remaining}, nil
 	}
 
-	digest, err := r.TagResolver.Resolve(ctx, pool.Spec.Image.Ref)
+	digest, err := r.TagResolver.Resolve(ctx, pool.Spec.Image.Ref, secretData)
 	if err != nil {
 		log.Error(err, "Failed to resolve tag", "ref", pool.Spec.Image.Ref)
 		setPoolDegraded(pool, bootcv1alpha1.PoolTagResolutionError, err.Error())
@@ -600,6 +652,38 @@ func (r *BootcNodePoolReconciler) listAllBootcNodes(
 	return all, nil
 }
 
+// fetchPullSecretData fetches the .dockerconfigjson data from the pull
+// secret referenced by the pool. Returns nil if no pull secret is set.
+func (r *BootcNodePoolReconciler) fetchPullSecretData(
+	ctx context.Context,
+	pool *bootcv1alpha1.BootcNodePool,
+) ([]byte, error) {
+	if pool.Spec.PullSecretRef == nil {
+		return nil, nil
+	}
+
+	key := types.NamespacedName{
+		Name:      pool.Spec.PullSecretRef.Name,
+		Namespace: pool.Spec.PullSecretRef.Namespace,
+	}
+	var secret corev1.Secret
+	if err := r.Get(ctx, key, &secret); err != nil {
+		return nil, fmt.Errorf("fetching pull secret %s/%s: %w", key.Namespace, key.Name, err)
+	}
+
+	data, ok := secret.Data[corev1.DockerConfigJsonKey]
+	if !ok {
+		return nil, fmt.Errorf(
+			"pull secret %s/%s missing %s key",
+			key.Namespace,
+			key.Name,
+			corev1.DockerConfigJsonKey,
+		)
+	}
+
+	return data, nil
+}
+
 // syncBootcNodeSpec updates a BootcNode's spec fields to match the pool.
 func (r *BootcNodePoolReconciler) syncBootcNodeSpec(
 	ctx context.Context,
@@ -786,10 +870,11 @@ func (r *BootcNodePoolReconciler) restoreCordonState(
 var poolDegradedPrecedence = map[string]int{
 	bootcv1alpha1.PoolHealthy:            0,
 	bootcv1alpha1.PoolNodeDegraded:       1,
-	bootcv1alpha1.PoolNodeConflict:       2,
-	bootcv1alpha1.PoolRolloutHalted:      3,
-	bootcv1alpha1.PoolInvalidSpec:        4,
-	bootcv1alpha1.PoolTagResolutionError: 5,
+	bootcv1alpha1.PoolSecretError:        2,
+	bootcv1alpha1.PoolNodeConflict:       3,
+	bootcv1alpha1.PoolRolloutHalted:      4,
+	bootcv1alpha1.PoolInvalidSpec:        5,
+	bootcv1alpha1.PoolTagResolutionError: 6,
 }
 
 // setPoolDegraded sets the Degraded condition on the pool, but only if the new

internal/controller/crd_test.go

@@ -27,7 +27,6 @@ const (
 
 	testSecretName = "my-pull-secret"
 	testSecretNS   = "bootc-operator"
-	testSecretHash = "sha256:b37e50cedcd3e3f1ff64f4afc0422084ae694253cf399326868e07a35f4a45fb" // "secret"
 )
 
 func TestBootcNodePoolCRD(t *testing.T) {
@@ -63,7 +62,7 @@ func TestBootcNodeCRD(t *testing.T) {
 	ctx := context.Background()
 
 	node := testutil.NewNode("worker-1", testImageDigestRefA,
-		testutil.WithNodePullSecret(testSecretName, testSecretNS, testSecretHash),
+		testutil.WithNodePullSecret(testSecretName, testSecretNS),
 	)
 
 	// Save the spec before Create, which mutates node in-place.