|
| 1 | +#!/usr/bin/env bats |
| 2 | +# Asserts that the sandbox Kubernetes cluster CIDR overlaps none of the address |
| 3 | +# ranges the sandbox platform is configured with (pod, service and join). |
| 4 | +# |
| 5 | +# Two allocators are live on this datapath and neither knows about the other. |
| 6 | +# kube-ovn owns pod IPAM and allocates flat from POD_CIDR across all nodes. |
| 7 | +# Cilium allocates nothing for pods -- it is chained behind kube-ovn -- but it |
| 8 | +# does carve its own per-node infrastructure addresses out of |
| 9 | +# Node.spec.podCIDR, which kube-controller-manager slices from the cluster |
| 10 | +# CIDR: the router IP on cilium_host, and the per-node Ingress IP that Gateway |
| 11 | +# API brings with it. Let the two ranges overlap and kube-ovn eventually hands |
| 12 | +# a pod an address cilium already holds. |
| 13 | +# |
| 14 | +# On the Ingress IP that is loud: the agent refuses the sandbox with "IP |
| 15 | +# ipv4:<addr> is already in use". Cilium never relents on the address, so the |
| 16 | +# pod recovers only if something puts it on a different one. Whether that |
| 17 | +# happens is not a property of this collision: hack/e2e-cilium-endpoint-leak- |
| 18 | +# healer.sh records deleting the wedged pod landing it on a clean address, and |
| 19 | +# in the failure this guard comes from the same address was refused seven times |
| 20 | +# across six distinct pod UIDs before the suite ran out of budget. On the |
| 21 | +# router IP it is silent instead: the endpoint is created, the pod runs, and |
| 22 | +# only host-to-pod traffic breaks, because to the node's own kernel that |
| 23 | +# address is local. |
| 24 | +# |
| 25 | +# Cilium picks those offsets by random scan over the node's range, so excluding |
| 26 | +# a fixed set of addresses on the kube-ovn side does not work -- the ranges |
| 27 | +# have to be disjoint. |
| 28 | +# |
| 29 | +# The invariant is checked, not the literals: any of the values may move, and |
| 30 | +# this still passes as long as none of them overlap. An extraction that comes |
| 31 | +# up empty, duplicated or misshapen fails the test instead of shrinking the |
| 32 | +# comparison, and comments and blank lines inside the list are skipped rather |
| 33 | +# than read as the end of it. This is not a YAML parser, though. Anything it |
| 34 | +# does not read as a bare dotted quad fails the shape validation rather than |
| 35 | +# being understood, and that set is wider than the exotic shapes: an anchor, a |
| 36 | +# nested flow mapping, but also a plainly quoted scalar. Quoting is asymmetric |
| 37 | +# between the two sides as well -- a platform key is read only when its value |
| 38 | +# is double-quoted, a Talos entry only when its value is not. All of those are |
| 39 | +# legal reformats and all of them turn the guard red with the offending value |
| 40 | +# in the message, which is the safe direction but not the same promise. |
| 41 | +# |
| 42 | +# Harness note: the CI path is hack/cozytest.sh, NOT real bats. There is no |
| 43 | +# `run`, `$status`, `$output`, `skip`, or setup()/teardown(); each test runs as |
| 44 | +# a shell function under `set -eu -x`, so a non-zero exit is the failure. A |
| 45 | +# `!`-negated command never trips errexit, so every assertion below is written |
| 46 | +# as an explicit `if ... return 1`. A top-level helper function would have |
| 47 | +# `return 0` injected before its closing brace by the runner's awk, which is |
| 48 | +# why the arithmetic is inlined instead. Paths are repo-root-relative. |
| 49 | +# |
| 50 | +# Run with: hack/cozytest.sh hack/sandbox-cidr-disjoint.bats |
| 51 | + |
| 52 | +@test "sandbox cluster CIDR overlaps no platform range" { |
| 53 | + talos_file=hack/e2e-prepare-cluster.bats |
| 54 | + install_file=hack/e2e-install-cozystack.bats |
| 55 | + |
| 56 | + # Talos cluster.network.podSubnets: every list item under the key, including |
| 57 | + # ones separated from the first by a comment or a blank line, so a second |
| 58 | + # entry cannot slip past unexamined. An IPv6 entry does not get an overlap |
| 59 | + # check, it fails the shape validation below; teaching this guard dual-stack |
| 60 | + # means extending it, not just adding the value. |
| 61 | + cluster_cidrs=$(awk ' |
| 62 | + /^[[:space:]]*podSubnets:[[:space:]]*$/ { inlist = 1; next } |
| 63 | + inlist && /^[[:space:]]*-[[:space:]]+[^[:space:]]/ { print $2; next } |
| 64 | + # A comment or a blank line between two entries is not the end of the |
| 65 | + # list. Treating it as the end would drop every entry below it and still |
| 66 | + # leave a non-empty result, which is the one way this extraction could go |
| 67 | + # green while checking less than it says. The block comment that landed |
| 68 | + # above podSubnets alongside this guard makes that the likely shape rather |
| 69 | + # than a rare one; the guard does not require that comment, it just has to |
| 70 | + # survive it and whatever gets written next to it later. |
| 71 | + inlist && /^[[:space:]]*(#|$)/ { next } |
| 72 | + inlist { inlist = 0 } |
| 73 | + ' "$talos_file") |
| 74 | + |
| 75 | + if [ -z "$cluster_cidrs" ]; then |
| 76 | + echo "found no podSubnets entries in $talos_file" >&2 |
| 77 | + echo "the extraction below is keyed on 'podSubnets:' followed by a list" >&2 |
| 78 | + return 1 |
| 79 | + fi |
| 80 | + |
| 81 | + # The ranges the install step hands the platform chart. podCIDR is the one |
| 82 | + # that collides today; serviceCIDR and joinCIDR are here because the cluster |
| 83 | + # CIDR must miss all three, and picking a replacement that lands on the join |
| 84 | + # range instead is the obvious next way to get this wrong. These three are |
| 85 | + # the allocator ranges and nothing else: the sandbox node network, the |
| 86 | + # load-balancer pool and the transit range of an off-by-default component |
| 87 | + # are not read here, so this list is not a complete map of what the cluster |
| 88 | + # CIDR must avoid. |
| 89 | + platform_cidrs= |
| 90 | + for key in podCIDR serviceCIDR joinCIDR; do |
| 91 | + # `|| true`: grep -c exits 1 when it counts zero, and under the runner's |
| 92 | + # errexit that kills the test on the assignment, before the message below |
| 93 | + # can name which key went missing. The outcome is red either way; this is |
| 94 | + # only about saying why. |
| 95 | + count=$(grep -cE "^[[:space:]]*${key}:[[:space:]]*\"" "$install_file" || true) |
| 96 | + # grep exits 2 with no stdout when the file itself is gone, which leaves |
| 97 | + # count empty and turns the arithmetic below into "integer expression |
| 98 | + # expected". Still red, but named after the wrong thing. |
| 99 | + count=${count:-0} |
| 100 | + if [ "$count" -ne 1 ]; then |
| 101 | + echo "expected exactly one $key key in $install_file, found $count" >&2 |
| 102 | + return 1 |
| 103 | + fi |
| 104 | + value=$(awk -F'"' -v k="$key" '$0 ~ "^[[:space:]]*" k ":[[:space:]]*\"" { print $2; exit }' "$install_file") |
| 105 | + # A present-but-empty value passes the count check above, then disappears |
| 106 | + # in the unquoted word split below, dropping that range from the |
| 107 | + # comparison with nothing to show for it. Refuse instead. |
| 108 | + if [ -z "$value" ]; then |
| 109 | + echo "$key in $install_file is present but empty" >&2 |
| 110 | + return 1 |
| 111 | + fi |
| 112 | + platform_cidrs="$platform_cidrs $value" |
| 113 | + done |
| 114 | + |
| 115 | + for cidr in $cluster_cidrs $platform_cidrs; do |
| 116 | + if printf '%s\n' "$cidr" | grep -qvE '^[0-9]{1,3}(\.[0-9]{1,3}){3}/[0-9]{1,2}$'; then |
| 117 | + echo "extracted value is not an IPv4 CIDR: '$cidr'" >&2 |
| 118 | + return 1 |
| 119 | + fi |
| 120 | + done |
| 121 | + |
| 122 | + for cluster_cidr in $cluster_cidrs; do |
| 123 | + for platform_cidr in $platform_cidrs; do |
| 124 | + # Two prefixes overlap when, masked to the shorter of the two lengths, |
| 125 | + # their network addresses are equal. Symmetric by construction: neither |
| 126 | + # argument is privileged, so containment is caught whichever way round |
| 127 | + # it is. awk arithmetic is IEEE double, exact well past the 2^32 needed. |
| 128 | + if printf '%s\n%s\n' "$cluster_cidr" "$platform_cidr" | awk ' |
| 129 | + function toint(a, p) { |
| 130 | + split(a, p, ".") |
| 131 | + return ((p[1] * 256 + p[2]) * 256 + p[3]) * 256 + p[4] |
| 132 | + } |
| 133 | + NR == 1 { split($0, a, "/"); ip1 = toint(a[1]); len1 = a[2] } |
| 134 | + NR == 2 { split($0, b, "/"); ip2 = toint(b[1]); len2 = b[2] } |
| 135 | + END { |
| 136 | + shorter = (len1 < len2 ? len1 : len2) |
| 137 | + block = 2 ^ (32 - shorter) |
| 138 | + exit (int(ip1 / block) == int(ip2 / block)) ? 0 : 1 |
| 139 | + } |
| 140 | + '; then |
| 141 | + echo "sandbox cluster CIDR $cluster_cidr ($talos_file)" >&2 |
| 142 | + echo "overlaps platform range $platform_cidr ($install_file)" >&2 |
| 143 | + echo "cilium carves its per-node router and Ingress addresses out of the cluster CIDR." >&2 |
| 144 | + echo "Pick a cluster CIDR that misses every range the platform is configured with." >&2 |
| 145 | + return 1 |
| 146 | + fi |
| 147 | + done |
| 148 | + done |
| 149 | +} |
0 commit comments