From 306cbc8390d46121e9cdcf603058b1a759c9f736 Mon Sep 17 00:00:00 2001 From: Saadi Myftija Date: Wed, 4 Feb 2026 12:50:19 +0100 Subject: [PATCH 1/2] feat(supervisor): project-based scheduling affinity for image cache locality Adds optional pod affinity so pods from the same project prefer scheduling on the same node. This can help improve image cache hit rates; subsequent pods benefit from already-pulled image layers, reducing startup time. Complements the built-in ImageLocality scheduler plugin by helping during burst scheduling scenarios. Pod affinity sees scheduled pods immediately, while ImageLocality only sees images after they're fully pulled. Configuration: - `KUBERNETES_PROJECT_AFFINITY_ENABLED` - Enable/disable (default: false) - `KUBERNETES_PROJECT_AFFINITY_WEIGHT` - Scheduler weight 1-100 (default: 50) - `KUBERNETES_PROJECT_AFFINITY_TOPOLOGY_KEY` - Topology key (default: kubernetes.io/hostname) Uses soft (preferred) affinity so pods always schedule even if preferred node is full. --- apps/supervisor/src/env.ts | 5 ++ .../src/workloadManager/kubernetes.ts | 88 +++++++++++++------ 2 files changed, 67 insertions(+), 26 deletions(-) diff --git a/apps/supervisor/src/env.ts b/apps/supervisor/src/env.ts index 9ef0cff253..9b46d5e664 100644 --- a/apps/supervisor/src/env.ts +++ b/apps/supervisor/src/env.ts @@ -112,6 +112,11 @@ const Env = z.object({ KUBERNETES_SCHEDULER_NAME: z.string().optional(), // Custom scheduler name for pods KUBERNETES_LARGE_MACHINE_POOL_LABEL: z.string().optional(), // if set, large-* presets affinity for machinepool= + // Project affinity settings - pods from the same project prefer the same node + KUBERNETES_PROJECT_AFFINITY_ENABLED: BoolEnv.default(false), + KUBERNETES_PROJECT_AFFINITY_WEIGHT: z.coerce.number().int().min(1).max(100).default(50), + KUBERNETES_PROJECT_AFFINITY_TOPOLOGY_KEY: z.string().default("kubernetes.io/hostname"), + // Placement tags settings PLACEMENT_TAGS_ENABLED: BoolEnv.default(false), PLACEMENT_TAGS_PREFIX: z.string().default("node.cluster.x-k8s.io"), diff --git a/apps/supervisor/src/workloadManager/kubernetes.ts b/apps/supervisor/src/workloadManager/kubernetes.ts index a725971a84..16c5eff9da 100644 --- a/apps/supervisor/src/workloadManager/kubernetes.ts +++ b/apps/supervisor/src/workloadManager/kubernetes.ts @@ -120,7 +120,7 @@ export class KubernetesWorkloadManager implements WorkloadManager { }, spec: { ...this.addPlacementTags(this.#defaultPodSpec, opts.placementTags), - affinity: this.#getNodeAffinity(opts.machine), + affinity: this.#getAffinity(opts.machine, opts.projectId), terminationGracePeriodSeconds: 60 * 60, containers: [ { @@ -390,7 +390,21 @@ export class KubernetesWorkloadManager implements WorkloadManager { return preset.name.startsWith("large-"); } - #getNodeAffinity(preset: MachinePreset): k8s.V1Affinity | undefined { + #getAffinity(preset: MachinePreset, projectId: string): k8s.V1Affinity | undefined { + const nodeAffinity = this.#getNodeAffinityRules(preset); + const podAffinity = this.#getProjectPodAffinity(projectId); + + if (!nodeAffinity && !podAffinity) { + return undefined; + } + + return { + ...(nodeAffinity && { nodeAffinity }), + ...(podAffinity && { podAffinity }), + }; + } + + #getNodeAffinityRules(preset: MachinePreset): k8s.V1NodeAffinity | undefined { if (!env.KUBERNETES_LARGE_MACHINE_POOL_LABEL) { return undefined; } @@ -398,42 +412,64 @@ export class KubernetesWorkloadManager implements WorkloadManager { if (this.#isLargeMachine(preset)) { // soft preference for the large-machine pool, falls back to standard if unavailable return { - nodeAffinity: { - preferredDuringSchedulingIgnoredDuringExecution: [ - { - weight: 100, - preference: { - matchExpressions: [ - { - key: "node.cluster.x-k8s.io/machinepool", - operator: "In", - values: [env.KUBERNETES_LARGE_MACHINE_POOL_LABEL], - }, - ], - }, + preferredDuringSchedulingIgnoredDuringExecution: [ + { + weight: 100, + preference: { + matchExpressions: [ + { + key: "node.cluster.x-k8s.io/machinepool", + operator: "In", + values: [env.KUBERNETES_LARGE_MACHINE_POOL_LABEL], + }, + ], }, - ], - }, + }, + ], }; } // not schedulable in the large-machine pool return { - nodeAffinity: { - requiredDuringSchedulingIgnoredDuringExecution: { - nodeSelectorTerms: [ - { + requiredDuringSchedulingIgnoredDuringExecution: { + nodeSelectorTerms: [ + { + matchExpressions: [ + { + key: "node.cluster.x-k8s.io/machinepool", + operator: "NotIn", + values: [env.KUBERNETES_LARGE_MACHINE_POOL_LABEL], + }, + ], + }, + ], + }, + }; + } + + #getProjectPodAffinity(projectId: string): k8s.V1PodAffinity | undefined { + if (!env.KUBERNETES_PROJECT_AFFINITY_ENABLED) { + return undefined; + } + + return { + preferredDuringSchedulingIgnoredDuringExecution: [ + { + weight: env.KUBERNETES_PROJECT_AFFINITY_WEIGHT, + podAffinityTerm: { + labelSelector: { matchExpressions: [ { - key: "node.cluster.x-k8s.io/machinepool", - operator: "NotIn", - values: [env.KUBERNETES_LARGE_MACHINE_POOL_LABEL], + key: "project", + operator: "In", + values: [projectId], }, ], }, - ], + topologyKey: env.KUBERNETES_PROJECT_AFFINITY_TOPOLOGY_KEY, + }, }, - }, + ], }; } } From 9c5be34c80904ba7c8a4b14044d99280cde5c148 Mon Sep 17 00:00:00 2001 From: Saadi Myftija Date: Wed, 4 Feb 2026 14:38:57 +0100 Subject: [PATCH 2/2] Add empty string guard for KUBERNETES_PROJECT_AFFINITY_TOPOLOGY_KEY --- apps/supervisor/src/env.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/supervisor/src/env.ts b/apps/supervisor/src/env.ts index 9b46d5e664..faf34bcd02 100644 --- a/apps/supervisor/src/env.ts +++ b/apps/supervisor/src/env.ts @@ -115,7 +115,7 @@ const Env = z.object({ // Project affinity settings - pods from the same project prefer the same node KUBERNETES_PROJECT_AFFINITY_ENABLED: BoolEnv.default(false), KUBERNETES_PROJECT_AFFINITY_WEIGHT: z.coerce.number().int().min(1).max(100).default(50), - KUBERNETES_PROJECT_AFFINITY_TOPOLOGY_KEY: z.string().default("kubernetes.io/hostname"), + KUBERNETES_PROJECT_AFFINITY_TOPOLOGY_KEY: z.string().trim().min(1).default("kubernetes.io/hostname"), // Placement tags settings PLACEMENT_TAGS_ENABLED: BoolEnv.default(false),