Skip to content

Commit 17c8b7b

Browse files
committed
test(spiffe): verify broker authorization and credential rotation
Signed-off-by: Taylor Mutch <[email protected]>
1 parent 89aad0f commit 17c8b7b

4 files changed

Lines changed: 905 additions & 0 deletions

File tree

Lines changed: 253 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,253 @@
1+
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2+
// SPDX-License-Identifier: Apache-2.0
3+
4+
//! Local SPIFFE Broker API experiment. See e2e/kubernetes/spiffe-broker/README.md.
5+
//! Credentials are short-lived test artifacts; no token or key is printed.
6+
7+
use std::{collections::HashMap, error::Error, path::PathBuf, sync::Arc, time::Duration};
8+
9+
use hyper_util::rt::TokioIo;
10+
use prost::Message;
11+
use rustls::{
12+
RootCertStore,
13+
pki_types::{CertificateDer, PrivateKeyDer, ServerName, pem::PemObject},
14+
};
15+
use tonic::{
16+
Request,
17+
transport::{Channel, Endpoint},
18+
};
19+
20+
type ProbeResult<T> = Result<T, Box<dyn Error + Send + Sync>>;
21+
22+
// Wire fields from spiffe/spiffe standards/brokerapi.proto. This probe uses
23+
// only the JWT profile and KubernetesObjectReference, never PID references.
24+
#[derive(Clone, PartialEq, Message)]
25+
struct AnyReference {
26+
#[prost(string, tag = "1")]
27+
type_url: String,
28+
#[prost(bytes = "vec", tag = "2")]
29+
value: Vec<u8>,
30+
}
31+
#[derive(Clone, PartialEq, Message)]
32+
struct WorkloadReference {
33+
#[prost(message, optional, tag = "1")]
34+
reference: Option<AnyReference>,
35+
}
36+
#[derive(Clone, PartialEq, Message)]
37+
struct ObjectType {
38+
#[prost(string, tag = "1")]
39+
plural: String,
40+
#[prost(string, tag = "2")]
41+
group: String,
42+
}
43+
#[derive(Clone, PartialEq, Message)]
44+
struct ObjectKey {
45+
#[prost(string, tag = "1")]
46+
namespace: String,
47+
#[prost(string, tag = "2")]
48+
name: String,
49+
}
50+
#[derive(Clone, PartialEq, Message)]
51+
struct ObjectReference {
52+
#[prost(message, optional, tag = "1")]
53+
r#type: Option<ObjectType>,
54+
#[prost(message, optional, tag = "2")]
55+
key: Option<ObjectKey>,
56+
#[prost(string, tag = "3")]
57+
uid: String,
58+
}
59+
#[derive(Clone, PartialEq, Message)]
60+
struct FetchRequest {
61+
#[prost(message, optional, tag = "1")]
62+
reference: Option<WorkloadReference>,
63+
#[prost(string, repeated, tag = "2")]
64+
audience: Vec<String>,
65+
#[prost(string, tag = "3")]
66+
spiffe_id: String,
67+
}
68+
#[derive(Clone, PartialEq, Message)]
69+
struct JwtSvid {
70+
#[prost(string, tag = "1")]
71+
spiffe_id: String,
72+
#[prost(string, tag = "2")]
73+
svid: String,
74+
#[prost(string, tag = "3")]
75+
hint: String,
76+
}
77+
#[derive(Clone, PartialEq, Message)]
78+
struct FetchResponse {
79+
#[prost(message, repeated, tag = "1")]
80+
svids: Vec<JwtSvid>,
81+
}
82+
#[derive(Clone, PartialEq, Message)]
83+
struct BundlesRequest {
84+
#[prost(message, optional, tag = "1")]
85+
reference: Option<WorkloadReference>,
86+
}
87+
#[derive(Clone, PartialEq, Message)]
88+
struct BundlesResponse {
89+
#[prost(map = "string, bytes", tag = "1")]
90+
bundles: HashMap<String, Vec<u8>>,
91+
}
92+
93+
fn env(name: &str) -> ProbeResult<String> {
94+
Ok(std::env::var(name)?)
95+
}
96+
fn request<T>(message: T) -> Request<T> {
97+
let mut request = Request::new(message);
98+
if std::env::var_os("BROKER_OMIT_HEADER").is_none() {
99+
request
100+
.metadata_mut()
101+
.insert("broker.spiffe.io", "true".parse().expect("static metadata"));
102+
}
103+
request
104+
}
105+
106+
async fn channel() -> ProbeResult<Channel> {
107+
let directory = PathBuf::from(env("BROKER_CREDENTIALS")?);
108+
let certificates = CertificateDer::pem_file_iter(directory.join("svid.0.pem"))?
109+
.collect::<Result<Vec<_>, _>>()?;
110+
let key = PrivateKeyDer::from_pem_file(directory.join("svid.0.key"))?;
111+
let mut roots = RootCertStore::empty();
112+
if std::env::var_os("BROKER_EMPTY_TRUST").is_none() {
113+
for certificate in CertificateDer::pem_file_iter(directory.join("bundle.0.pem"))? {
114+
roots.add(certificate?)?;
115+
}
116+
}
117+
let mut tls = openshell_crypto::tls::try_client_builder()?
118+
.dangerous()
119+
.with_custom_certificate_verifier(openshell_crypto::tls::spiffe::server_verifier(
120+
roots,
121+
env("BROKER_AGENT_ID")?,
122+
))
123+
.with_client_auth_cert(certificates, key)?;
124+
tls.alpn_protocols = vec![b"h2".to_vec()];
125+
let connector = tokio_rustls::TlsConnector::from(Arc::new(tls));
126+
let address = env("BROKER_ADDRESS")?;
127+
Ok(Endpoint::from_static("http://broker.invalid")
128+
.connect_timeout(Duration::from_secs(10))
129+
.timeout(Duration::from_secs(10))
130+
.connect_with_connector(tower::service_fn(move |_| {
131+
let connector = connector.clone();
132+
let address = address.clone();
133+
async move {
134+
let tcp = openshell_core::net::connect_tcp_nodelay_best_effort(address).await?;
135+
let stream = connector
136+
.connect(
137+
ServerName::try_from("broker.invalid").expect("static name"),
138+
tcp,
139+
)
140+
.await?;
141+
Ok::<_, std::io::Error>(TokioIo::new(stream))
142+
}
143+
}))
144+
.await?)
145+
}
146+
147+
async fn probe() -> ProbeResult<()> {
148+
let reference = WorkloadReference {
149+
reference: Some(AnyReference {
150+
type_url: "type.googleapis.com/spiffe.broker.KubernetesObjectReference".into(),
151+
value: ObjectReference {
152+
r#type: Some(ObjectType {
153+
plural: env("BROKER_RESOURCE")?,
154+
group: env("BROKER_GROUP")?,
155+
}),
156+
key: if std::env::var_os("BROKER_UID_ONLY").is_some() {
157+
None
158+
} else {
159+
Some(ObjectKey {
160+
namespace: env("BROKER_NAMESPACE")?,
161+
name: env("BROKER_NAME")?,
162+
})
163+
},
164+
uid: env("BROKER_UID")?,
165+
}
166+
.encode_to_vec(),
167+
}),
168+
};
169+
let expected = env("BROKER_SANDBOX_ID")?;
170+
let audience = "openshell-broker-probe";
171+
let mut grpc = tonic::client::Grpc::new(channel().await?);
172+
grpc.ready()
173+
.await
174+
.map_err(|_| "broker channel unavailable")?;
175+
let response: Result<tonic::Response<FetchResponse>, _> = grpc
176+
.unary(
177+
request(FetchRequest {
178+
reference: Some(reference.clone()),
179+
audience: vec![audience.into()],
180+
spiffe_id: expected.clone(),
181+
}),
182+
http::uri::PathAndQuery::from_static("/spiffe.broker.API/FetchJWTSVID"),
183+
tonic_prost::ProstCodec::default(),
184+
)
185+
.await;
186+
if let Ok(code) = std::env::var("BROKER_EXPECT_ERROR") {
187+
match response {
188+
Err(status) if format!("{:?}", status.code()) == code => {
189+
println!("PASS: request rejected with {code}");
190+
return Ok(());
191+
}
192+
Err(status) => {
193+
return Err(format!("expected {code}; received {:?}", status.code()).into());
194+
}
195+
Ok(_) => return Err("expected rejection; issuance succeeded".into()),
196+
}
197+
}
198+
let response = response
199+
.map_err(|status| format!("issuance failed: {:?}", status.code()))?
200+
.into_inner();
201+
let [svid] = response.svids.as_slice() else {
202+
return Err("expected exactly one JWT-SVID".into());
203+
};
204+
if svid.spiffe_id != expected {
205+
return Err("unexpected response identity".into());
206+
}
207+
grpc.ready()
208+
.await
209+
.map_err(|_| "broker channel unavailable")?;
210+
let mut stream: tonic::Streaming<BundlesResponse> = grpc
211+
.server_streaming(
212+
request(BundlesRequest {
213+
reference: Some(reference),
214+
}),
215+
http::uri::PathAndQuery::from_static("/spiffe.broker.API/SubscribeToJWTBundles"),
216+
tonic_prost::ProstCodec::default(),
217+
)
218+
.await
219+
.map_err(|status| format!("bundles failed: {:?}", status.code()))?
220+
.into_inner();
221+
let bundles = stream
222+
.message()
223+
.await
224+
.map_err(|status| format!("bundles failed: {:?}", status.code()))?
225+
.ok_or("empty bundle stream")?;
226+
let domain = spiffe::TrustDomain::new(
227+
openshell_spiffe::trust_domain(&expected).ok_or("invalid expected identity")?,
228+
)?;
229+
let bundle = spiffe::JwtBundle::from_jwt_authorities(
230+
domain.clone(),
231+
bundles
232+
.bundles
233+
.get(&domain.id_string())
234+
.ok_or("missing trust-domain bundle")?,
235+
)?;
236+
let mut bundle_set = spiffe::JwtBundleSet::new();
237+
bundle_set.add_bundle(bundle);
238+
openshell_crypto::install_jwt_provider();
239+
let verified = spiffe::JwtSvid::parse_and_validate(&svid.svid, &bundle_set, &[audience])?;
240+
if verified.spiffe_id().to_string() != expected {
241+
return Err("unexpected verified identity".into());
242+
}
243+
println!(
244+
"PASS: verified sandbox JWT-SVID {expected}; expires_at={}",
245+
verified.claims().exp()
246+
);
247+
Ok(())
248+
}
249+
250+
#[tokio::main]
251+
async fn main() -> ProbeResult<()> {
252+
tokio::time::timeout(Duration::from_secs(30), probe()).await?
253+
}
Lines changed: 88 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,88 @@
1+
# SPIFFE Broker API experiment
2+
3+
This local spike checks whether SPIRE's Broker API can issue the stable sandbox
4+
JWT-SVID over mTLS using an attested Kubernetes object reference. It runs
5+
against the existing Helm dev cluster in the checkout's `kubeconfig`, with the
6+
SPIRE 1.15.3 / hardened chart 0.30.3 fixture already installed in `spire`.
7+
The gateway integration is covered by the separate end-to-end task below.
8+
9+
Run from the repository root:
10+
11+
```shell
12+
mise exec -- cargo build -p openshell-spire --example broker_probe
13+
mise exec -- uv run --no-project python e2e/kubernetes/spiffe-broker/experiment.py setup
14+
mise exec -- uv run --no-project python e2e/kubernetes/spiffe-broker/experiment.py test
15+
```
16+
17+
`setup` backs up the agent ConfigMap under ignored `.cache/spiffe-broker/`,
18+
enables the experimental TCP Broker endpoint on port 8443, adds agent permissions
19+
to resolve Sandbox objects and perform SubjectAccessReviews, and restarts the
20+
agent. Use a dedicated development cluster without concurrent SPIRE configuration
21+
changes. The script expects the fixture's resource names and `openshell.local`
22+
trust domain.
23+
24+
The two client pods run in `openshell-broker-probe`, where Restricted Pod Security
25+
is enforced. Their only identity socket is the CSI Workload API mount. An init
26+
container obtains a short-lived X.509-SVID; the script copies the test credentials
27+
to ignored files with mode 0600 for the native Rust probe. This export is a test
28+
shortcut. A gateway integration must obtain and rotate its own credentials through
29+
the Workload API without exporting them to the operator's machine.
30+
31+
`test` owns a temporary port-forward from localhost:18443 to the agent. The Rust
32+
client verifies the agent's certificate chain and exact SPIFFE URI SAN using
33+
`openshell-crypto`. It sends the required `broker.spiffe.io: true` header,
34+
requests a JWT-SVID, retrieves the JWT bundle, then verifies the signature,
35+
audience, expiry, and expected sandbox subject. Tokens and private keys are not
36+
printed.
37+
38+
The target is a suspended Sandbox CR referenced by resource type, namespace,
39+
name, and UID. Its SPIRE registration matches `k8s:resource` and `k8s:uid`,
40+
using properties attested by SPIRE. The client's SPIFFE
41+
identity has the custom `impersonate-via-spire` permission for only the allowed
42+
Sandbox. A second Sandbox exists to check authorization denial.
43+
44+
## Observed results
45+
46+
Against the local k3d Helm dev cluster on November 13, 2026:
47+
48+
| Check | Result |
49+
| --- | --- |
50+
| Allowed Sandbox reference | JWT-SVID issued and verified |
51+
| UID-only Sandbox reference | JWT-SVID issued and verified |
52+
| Missing Broker metadata header | `InvalidArgument` |
53+
| Existing Sandbox outside the client's object grant | `PermissionDenied` |
54+
| Correct name with a stale UID | `NotFound` |
55+
| Valid X.509-SVID absent from the broker allowlist | Request refused; native client observes `Unknown` |
56+
| Agent certificate with an empty trust store | Client rejects the certificate chain |
57+
| Unexpected agent SPIFFE identity | Client rejects the TLS certificate |
58+
59+
This establishes transport and identity issuance against real SPIRE with enforced
60+
Kubernetes authorization. It does not establish OpenShift operator support,
61+
credential rotation, multi-agent routing, or the full gateway provider-token and
62+
supervisor-failover path.
63+
64+
## Gateway verification
65+
66+
The branch's gateway adapter uses the SPIFFE Broker API with the persisted
67+
Sandbox CR UID. Run the complete provider-token and failover path with:
68+
69+
```shell
70+
mise run e2e:kubernetes:shared-supervisor-spiffe
71+
```
72+
73+
That task creates a separate test cluster and pinned SPIRE installation. It
74+
also tests Restricted gateway admission and acquisition after X.509 credential
75+
rotation. The independent probe above targets the existing dev cluster.
76+
77+
Restore the agent's original configuration and remove the probe objects and
78+
exported credentials:
79+
80+
```shell
81+
mise exec -- uv run --no-project python e2e/kubernetes/spiffe-broker/experiment.py cleanup
82+
```
83+
84+
Upstream contracts:
85+
86+
- [SPIRE 1.15.3 Broker API](https://github.com/spiffe/spire/blob/v1.15.3/doc/spire_agent.md#spiffe-broker-api)
87+
- [SPIRE 1.15.3 Kubernetes reference attestation](https://github.com/spiffe/spire/blob/v1.15.3/doc/plugin_agent_workloadattestor_k8s.md)
88+
- [Broker API protobuf](https://github.com/spiffe/spiffe/blob/main/standards/brokerapi.proto)

0 commit comments

Comments
 (0)