Skip to content

Commit 1d4cf7f

Browse files
Merge pull request NVIDIA#22 from markatcanonical/deploy-k8s
Update the `cluster init` command to handle registry authentication for K8s deployment
2 parents a0c2686 + 5ce7138 commit 1d4cf7f

4 files changed

Lines changed: 203 additions & 22 deletions

File tree

‎crates/openshell-bootstrap/src/docker.rs‎

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -701,11 +701,13 @@ pub async fn ensure_container(
701701
.filter(|u| !u.is_empty())
702702
.map(ToString::to_string)
703703
});
704-
let effective_password = env_non_empty("OPENSHELL_REGISTRY_PASSWORD").or_else(|| {
705-
registry_token
706-
.filter(|t| !t.is_empty())
707-
.map(ToString::to_string)
708-
});
704+
let effective_password = env_non_empty("OPENSHELL_REGISTRY_TOKEN")
705+
.or_else(|| env_non_empty("OPENSHELL_REGISTRY_PASSWORD"))
706+
.or_else(|| {
707+
registry_token
708+
.filter(|t| !t.is_empty())
709+
.map(ToString::to_string)
710+
});
709711

710712
let mut env_vars: Vec<String> = vec![
711713
format!("REGISTRY_MODE={REGISTRY_MODE_EXTERNAL}"),

‎crates/openshell-bootstrap/src/k8s_init.rs‎

Lines changed: 107 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,14 @@ use crate::constants::{
1919
use crate::pki;
2020
use std::io::Read;
2121

22-
pub async fn init_external_cluster(namespace: &str, registry: &str) -> Result<()> {
22+
pub async fn init_external_cluster(
23+
gateway_name: &str,
24+
namespace: &str,
25+
registry: &str,
26+
registry_username: Option<&str>,
27+
registry_token: Option<&str>,
28+
registry_authfile: Option<&str>,
29+
) -> Result<()> {
2330
let client = Client::try_default()
2431
.await
2532
.into_diagnostic()
@@ -105,12 +112,82 @@ pub async fn init_external_cluster(namespace: &str, registry: &str) -> Result<()
105112

106113
let snap_user_data = std::env::var("SNAP_USER_DATA").unwrap_or_else(|_| "/tmp".to_string());
107114

115+
let (authfile_path, _tmp_dir_guard) = if let Some(path) = registry_authfile {
116+
(std::path::PathBuf::from(path), None)
117+
} else {
118+
// Validate and resolve credentials
119+
let effective_username = match (registry_username, registry_token) {
120+
(Some(u), Some(_)) => Some(u),
121+
(None, Some(_)) => Some("__token__"),
122+
(Some(_), None) => {
123+
return Err(miette::miette!(
124+
"A registry token/password must be provided when a username is specified."
125+
));
126+
}
127+
(None, None) => None,
128+
};
129+
130+
if let Some(token) = registry_token {
131+
// Create tempdir for secure skopeo execution
132+
let tmp_dir = tempfile::tempdir()
133+
.into_diagnostic()
134+
.context("Failed to create temporary directory for secure auth")?;
135+
let authfile = tmp_dir.path().join("auth.json");
136+
137+
let u = effective_username.unwrap();
138+
let mut login_cmd = std::process::Command::new("skopeo");
139+
login_cmd
140+
.arg("login")
141+
.arg("--authfile")
142+
.arg(authfile.as_os_str())
143+
.arg("--username")
144+
.arg(u)
145+
.arg("--password-stdin")
146+
.arg(target_registry);
147+
148+
// Pipe token to stdin
149+
login_cmd.stdin(std::process::Stdio::piped());
150+
login_cmd.stdout(std::process::Stdio::null());
151+
login_cmd.stderr(std::process::Stdio::piped());
152+
153+
let mut child = login_cmd
154+
.spawn()
155+
.into_diagnostic()
156+
.context("Failed to spawn skopeo login")?;
157+
if let Some(mut stdin) = child.stdin.take() {
158+
use std::io::Write;
159+
stdin.write_all(token.as_bytes()).into_diagnostic()?;
160+
}
161+
let status = child
162+
.wait()
163+
.into_diagnostic()
164+
.context("Failed to wait on skopeo login")?;
165+
if !status.success() {
166+
return Err(miette::miette!(
167+
"Failed to authenticate with registry using skopeo login"
168+
));
169+
}
170+
(authfile, Some(tmp_dir))
171+
} else {
172+
(std::path::PathBuf::new(), None)
173+
}
174+
};
175+
176+
let has_authfile = registry_authfile.is_some() || registry_token.is_some();
177+
108178
if is_core_rock {
109179
let target_image = format!("{}/openshell-core:local", target_registry);
110-
let status = std::process::Command::new("skopeo")
111-
.arg("--insecure-policy")
180+
181+
let mut cmd = std::process::Command::new("skopeo");
182+
cmd.arg("--insecure-policy")
112183
.arg("copy")
113-
.arg("--dest-tls-verify=false")
184+
.arg("--dest-tls-verify=false");
185+
186+
if has_authfile {
187+
cmd.arg("--authfile").arg(authfile_path.as_os_str());
188+
}
189+
190+
let status = cmd
114191
.arg(format!("--tmpdir={}", snap_user_data))
115192
.arg(format!("oci-archive:{}", core_rock))
116193
.arg(format!("docker://{}", target_image))
@@ -129,21 +206,33 @@ pub async fn init_external_cluster(namespace: &str, registry: &str) -> Result<()
129206
let target_gateway = format!("{}/openshell-gateway:dev", target_registry);
130207
let target_supervisor = format!("{}/openshell-supervisor:dev", target_registry);
131208

132-
let s1 = std::process::Command::new("skopeo")
133-
.arg("--insecure-policy")
209+
let mut cmd1 = std::process::Command::new("skopeo");
210+
cmd1.arg("--insecure-policy")
134211
.arg("copy")
135-
.arg("--dest-tls-verify=false")
212+
.arg("--dest-tls-verify=false");
213+
214+
if has_authfile {
215+
cmd1.arg("--authfile").arg(authfile_path.as_os_str());
216+
}
217+
218+
let s1 = cmd1
136219
.arg(format!("--tmpdir={}", snap_user_data))
137220
.arg(format!("oci-archive:{}", gateway_tar))
138221
.arg(format!("docker://{}", target_gateway))
139222
.stdout(std::process::Stdio::null())
140223
.stderr(std::process::Stdio::piped())
141224
.status();
142225

143-
let s2 = std::process::Command::new("skopeo")
144-
.arg("--insecure-policy")
226+
let mut cmd2 = std::process::Command::new("skopeo");
227+
cmd2.arg("--insecure-policy")
145228
.arg("copy")
146-
.arg("--dest-tls-verify=false")
229+
.arg("--dest-tls-verify=false");
230+
231+
if has_authfile {
232+
cmd2.arg("--authfile").arg(authfile_path.as_os_str());
233+
}
234+
235+
let s2 = cmd2
147236
.arg(format!("--tmpdir={}", snap_user_data))
148237
.arg(format!("oci-archive:{}", supervisor_tar))
149238
.arg(format!("docker://{}", target_supervisor))
@@ -183,7 +272,10 @@ pub async fn init_external_cluster(namespace: &str, registry: &str) -> Result<()
183272
.arg("gateway")
184273
.arg(&chart_path)
185274
.arg("--namespace")
186-
.arg(namespace);
275+
.arg(namespace)
276+
.arg("--wait")
277+
.arg("--timeout")
278+
.arg("2m");
187279

188280
if images_loaded {
189281
if is_core_rock {
@@ -244,7 +336,7 @@ pub async fn init_external_cluster(namespace: &str, registry: &str) -> Result<()
244336
.join(".config")
245337
.join("openshell")
246338
.join("gateways")
247-
.join(namespace)
339+
.join(gateway_name)
248340
.join("mtls");
249341
std::fs::create_dir_all(&mtls_dir).into_diagnostic()?;
250342

@@ -254,10 +346,10 @@ pub async fn init_external_cluster(namespace: &str, registry: &str) -> Result<()
254346

255347
// 7. Save gateway metadata and set active gateway
256348
let metadata = crate::metadata::create_gateway_metadata(
257-
namespace, None, 30051, // NodePort from gateway.yaml
349+
gateway_name, None, 30051, // NodePort from gateway.yaml
258350
);
259-
crate::metadata::store_gateway_metadata(namespace, &metadata)?;
260-
crate::metadata::save_active_gateway(namespace)?;
351+
crate::metadata::store_gateway_metadata(gateway_name, &metadata)?;
352+
crate::metadata::save_active_gateway(gateway_name)?;
261353

262354
Ok(())
263355
}

‎crates/openshell-cli/Cargo.toml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,7 @@ url = { workspace = true }
7373
tracing = { workspace = true }
7474
tracing-subscriber = { workspace = true }
7575
tower.workspace = true
76+
tempfile = "3"
7677

7778
[lints]
7879
workspace = true

‎crates/openshell-cli/src/main.rs‎

Lines changed: 88 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -627,6 +627,23 @@ enum ClusterCommands {
627627
/// The destination registry for stashing rocks.
628628
#[arg(long, required = true)]
629629
registry: String,
630+
631+
/// Registry username for authenticating pushes.
632+
#[arg(long)]
633+
registry_username: Option<String>,
634+
635+
/// Registry token/password for authenticating pushes.
636+
#[arg(long)]
637+
registry_token: Option<String>,
638+
639+
/// Path to a registry authentication file (e.g. docker config.json).
640+
#[arg(long)]
641+
registry_authfile: Option<String>,
642+
643+
/// Path to the kubeconfig file, or '-' to read from stdin.
644+
/// If omitted, uses the KUBECONFIG environment variable or ~/.kube/config.
645+
#[arg(long)]
646+
kubeconfig: Option<String>,
630647
},
631648
}
632649

@@ -1752,8 +1769,77 @@ async fn main() -> Result<()> {
17521769
Some(Commands::Cluster {
17531770
command: Some(command),
17541771
}) => match command {
1755-
ClusterCommands::Init { namespace, registry } => {
1756-
openshell_bootstrap::k8s_init::init_external_cluster(&namespace, &registry).await?;
1772+
ClusterCommands::Init { namespace, registry, registry_username, registry_token, registry_authfile, kubeconfig } => {
1773+
let gateway_name = match cli.gateway {
1774+
Some(name) => name,
1775+
None => return Err(miette::miette!("ERROR: --gateway is required to name the new K8s gateway")),
1776+
};
1777+
1778+
let args: Vec<String> = std::env::args().collect();
1779+
if args.iter().any(|arg| arg == "--gateway-endpoint" || arg.starts_with("--gateway-endpoint=")) {
1780+
return Err(miette::miette!("ERROR: no endpoint needed for k8s deployment, only --kubeconfig"));
1781+
}
1782+
1783+
if openshell_bootstrap::load_gateway_metadata(&gateway_name).is_ok() {
1784+
return Err(miette::miette!("Gateway '{}' already exists.", gateway_name));
1785+
}
1786+
1787+
// Handle custom kubeconfig if provided
1788+
let mut _temp_kubeconfig: Option<tempfile::NamedTempFile> = None;
1789+
if let Some(config_path) = kubeconfig {
1790+
if config_path == "-" {
1791+
let mut buf = Vec::new();
1792+
std::io::Read::read_to_end(&mut std::io::stdin(), &mut buf)
1793+
.map_err(|e| miette::miette!("Failed to read kubeconfig from stdin: {}", e))?;
1794+
1795+
let mut temp = tempfile::NamedTempFile::new()
1796+
.map_err(|e| miette::miette!("Failed to create temporary file for kubeconfig: {}", e))?;
1797+
std::io::Write::write_all(&mut temp, &buf)
1798+
.map_err(|e| miette::miette!("Failed to write kubeconfig to temporary file: {}", e))?;
1799+
1800+
// Keep the file alive for the duration of this command
1801+
let path = temp.path().to_path_buf();
1802+
_temp_kubeconfig = Some(temp);
1803+
#[allow(unsafe_code)]
1804+
unsafe {
1805+
std::env::set_var("KUBECONFIG", path);
1806+
}
1807+
} else {
1808+
#[allow(unsafe_code)]
1809+
unsafe {
1810+
std::env::set_var("KUBECONFIG", config_path);
1811+
}
1812+
}
1813+
}
1814+
1815+
let effective_username = std::env::var("OPENSHELL_REGISTRY_USERNAME")
1816+
.ok()
1817+
.filter(|s| !s.is_empty())
1818+
.or(registry_username);
1819+
1820+
let effective_password = std::env::var("OPENSHELL_REGISTRY_TOKEN")
1821+
.ok()
1822+
.filter(|s| !s.is_empty())
1823+
.or_else(|| {
1824+
std::env::var("OPENSHELL_REGISTRY_PASSWORD")
1825+
.ok()
1826+
.filter(|s| !s.is_empty())
1827+
})
1828+
.or(registry_token);
1829+
1830+
let effective_authfile = std::env::var("OPENSHELL_REGISTRY_AUTHFILE")
1831+
.ok()
1832+
.filter(|s| !s.is_empty())
1833+
.or(registry_authfile);
1834+
1835+
openshell_bootstrap::k8s_init::init_external_cluster(
1836+
&gateway_name,
1837+
&namespace,
1838+
&registry,
1839+
effective_username.as_deref(),
1840+
effective_password.as_deref(),
1841+
effective_authfile.as_deref()
1842+
).await?;
17571843
println!("Cluster initialized successfully.");
17581844
}
17591845
},

0 commit comments

Comments
 (0)