Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions architecture/gateway-single-node.md
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,7 @@ Development task entrypoints split bootstrap behavior:

For `mise run cluster`, `.env` acts as local source-of-truth for `GATEWAY_NAME`, `GATEWAY_PORT`, and `OPENSHELL_GATEWAY`. Missing keys are appended; existing values are preserved. If `GATEWAY_PORT` is missing, the task selects a free local port and persists it.
Fast mode ensures a local registry (`127.0.0.1:5000`) is running and configures k3s to mirror pulls via `host.docker.internal:5000`, so the cluster task can push/pull local component images consistently.
Sandbox service routing is always configured for gateway deployments. Bootstrap derives a default service base domain from the gateway name, and fast deploy passes the same domain through Helm so incremental deploys do not reset it.

## Bootstrap Sequence Diagram

Expand Down
6 changes: 6 additions & 0 deletions crates/openshell-bootstrap/src/docker.rs
Original file line number Diff line number Diff line change
Expand Up @@ -493,6 +493,7 @@ pub async fn ensure_image(
/// because the container was originally created with a different port.
// Refactoring this signature would touch many call sites across the workspace.
#[allow(clippy::too_many_arguments)]
#[allow(clippy::fn_params_excessive_bools)]
pub async fn ensure_container(
docker: &Docker,
name: &str,
Expand All @@ -502,6 +503,7 @@ pub async fn ensure_container(
gateway_port: u16,
disable_tls: bool,
disable_gateway_auth: bool,
service_base_domains: &[String],
registry_username: Option<&str>,
registry_token: Option<&str>,
device_ids: &[String],
Expand Down Expand Up @@ -748,6 +750,10 @@ pub async fn ensure_container(
// gateway port for remote clusters must match.
env_vars.push(format!("SSH_GATEWAY_PORT={gateway_port}"));
}
env_vars.push(format!(
"SERVICE_BASE_DOMAINS={}",
service_base_domains.join(",")
));

// Pass image configuration to the cluster entrypoint.
// The effective tag is resolved from the runtime IMAGE_TAG env var (if set)
Expand Down
24 changes: 24 additions & 0 deletions crates/openshell-bootstrap/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,7 @@ impl RemoteOptions {
pub const DEFAULT_GATEWAY_PORT: u16 = 8080;

#[derive(Debug, Clone)]
#[allow(clippy::struct_excessive_bools)]
pub struct DeployOptions {
pub name: String,
pub image_ref: Option<String>,
Expand All @@ -104,6 +105,8 @@ pub struct DeployOptions {
/// Disable gateway authentication (mTLS client certificate requirement).
/// Ignored when `disable_tls` is true.
pub disable_gateway_auth: bool,
/// Base domains accepted for sandbox service routing.
pub service_base_domains: Vec<String>,
/// Registry authentication username. Defaults to `__token__` when a
/// `registry_token` is provided but no username is set. Only needed
/// for private registries — public GHCR repos pull without auth.
Expand Down Expand Up @@ -150,6 +153,7 @@ impl DeployOptions {
gateway_host: None,
disable_tls: false,
disable_gateway_auth: false,
service_base_domains: Vec::new(),
registry_username: None,
registry_token: None,
gpu: vec![],
Expand Down Expand Up @@ -199,6 +203,16 @@ impl DeployOptions {
self
}

#[must_use]
pub fn with_service_base_domains<I, S>(mut self, domains: I) -> Self
where
I: IntoIterator<Item = S>,
S: Into<String>,
{
self.service_base_domains = domains.into_iter().map(Into::into).collect();
self
}

/// Set the registry authentication username.
#[must_use]
pub fn with_registry_username(mut self, username: impl Into<String>) -> Self {
Expand Down Expand Up @@ -332,6 +346,11 @@ where
let gateway_host = options.gateway_host;
let disable_tls = options.disable_tls;
let disable_gateway_auth = options.disable_gateway_auth;
let service_base_domains = if options.service_base_domains.is_empty() {
vec![openshell_core::config::default_service_base_domain_for_gateway(&name)]
} else {
options.service_base_domains
};
let registry_username = options.registry_username;
let registry_token = options.registry_token;
let gpu = options.gpu;
Expand Down Expand Up @@ -475,6 +494,10 @@ where
{
sans.push(host.clone());
}
for base_domain in &service_base_domains {
sans.push(base_domain.clone());
sans.push(format!("*.{base_domain}"));
}
(sans, gateway_host)
};

Expand Down Expand Up @@ -524,6 +547,7 @@ where
port,
disable_tls,
disable_gateway_auth,
&service_base_domains,
registry_username.as_deref(),
registry_token.as_deref(),
&device_ids,
Expand Down
56 changes: 56 additions & 0 deletions crates/openshell-cli/src/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -200,6 +200,7 @@ const HELP_TEMPLATE: &str = "\

\x1b[1mSANDBOX COMMANDS\x1b[0m
sandbox: Manage sandboxes
service: Expose sandbox services
forward: Manage port forwarding to a sandbox
logs: View sandbox logs
policy: Manage sandbox policy
Expand Down Expand Up @@ -415,6 +416,13 @@ enum Commands {
command: Option<ForwardCommands>,
},

/// Expose sandbox services.
#[command(help_template = SUBCOMMAND_HELP_TEMPLATE)]
Service {
#[command(subcommand)]
command: Option<ServiceCommands>,
},

/// View sandbox logs.
#[command(alias = "lg", after_help = LOGS_EXAMPLES, help_template = LEAF_HELP_TEMPLATE, next_help_heading = "FLAGS")]
Logs {
Expand Down Expand Up @@ -826,6 +834,11 @@ enum GatewayCommands {
#[arg(long)]
disable_gateway_auth: bool,

/// Base domain accepted for sandbox service routes. May be repeated.
/// Defaults to `<gateway>.openshell.localhost`.
#[arg(long = "service-base-domain")]
service_base_domains: Vec<String>,

/// Username for authenticating with the container image registry.
///
/// Defaults to `__token__` when `--registry-token` is set (the
Expand Down Expand Up @@ -1774,6 +1787,24 @@ enum ForwardCommands {
List,
}

#[derive(Subcommand, Debug)]
enum ServiceCommands {
/// Expose an HTTP service running inside a sandbox.
#[command(help_template = LEAF_HELP_TEMPLATE, next_help_heading = "FLAGS")]
Expose {
/// Sandbox name.
#[arg(add = ArgValueCompleter::new(completers::complete_sandbox_names))]
sandbox: String,

/// Service name.
service: String,

/// Loopback TCP port inside the sandbox.
#[arg(long)]
target_port: u16,
},
}

#[tokio::main]
#[allow(clippy::large_stack_frames)] // CLI dispatch holds many futures; OK at top level.
async fn main() -> Result<()> {
Expand Down Expand Up @@ -1836,6 +1867,7 @@ async fn main() -> Result<()> {
recreate,
plaintext,
disable_gateway_auth,
service_base_domains,
registry_username,
registry_token,
gpu,
Expand All @@ -1862,6 +1894,7 @@ async fn main() -> Result<()> {
recreate,
plaintext,
disable_gateway_auth,
service_base_domains,
registry_username.as_deref(),
registry_token.as_deref(),
gpu,
Expand Down Expand Up @@ -2118,6 +2151,22 @@ async fn main() -> Result<()> {
}
},

// -----------------------------------------------------------
// Service exposure
// -----------------------------------------------------------
Some(Commands::Service {
command:
Some(ServiceCommands::Expose {
sandbox,
service,
target_port,
}),
}) => {
let ctx = resolve_gateway(&cli.gateway, &cli.gateway_endpoint)?;
let mut tls = tls.with_gateway_name(&ctx.name);
apply_auth(&mut tls, &ctx.name);
run::service_expose(&ctx.endpoint, &sandbox, &service, target_port, &tls).await?;
}
// -----------------------------------------------------------
// Top-level logs (was `sandbox logs`)
// -----------------------------------------------------------
Expand Down Expand Up @@ -2869,6 +2918,13 @@ async fn main() -> Result<()> {
.print_help()
.expect("Failed to print help");
}
Some(Commands::Service { command: None }) => {
Cli::command()
.find_subcommand_mut("service")
.expect("service subcommand exists")
.print_help()
.expect("Failed to print help");
}
Some(Commands::Policy { command: None }) => {
Cli::command()
.find_subcommand_mut("policy")
Expand Down
95 changes: 86 additions & 9 deletions crates/openshell-cli/src/run.rs
Original file line number Diff line number Diff line change
Expand Up @@ -25,13 +25,14 @@ use openshell_bootstrap::{
use openshell_core::proto::{
ApproveAllDraftChunksRequest, ApproveDraftChunkRequest, ClearDraftChunksRequest,
CreateProviderRequest, CreateSandboxRequest, DeleteProviderRequest, DeleteSandboxRequest,
ExecSandboxRequest, GetClusterInferenceRequest, GetDraftHistoryRequest, GetDraftPolicyRequest,
GetGatewayConfigRequest, GetProviderRequest, GetSandboxConfigRequest, GetSandboxLogsRequest,
GetSandboxPolicyStatusRequest, GetSandboxRequest, HealthRequest, ListProvidersRequest,
ListSandboxPoliciesRequest, ListSandboxesRequest, PolicySource, PolicyStatus, Provider,
RejectDraftChunkRequest, Sandbox, SandboxPhase, SandboxPolicy, SandboxSpec, SandboxTemplate,
SetClusterInferenceRequest, SettingScope, SettingValue, UpdateConfigRequest,
UpdateProviderRequest, WatchSandboxRequest, exec_sandbox_event, setting_value,
ExecSandboxRequest, ExposeServiceRequest, GetClusterInferenceRequest, GetDraftHistoryRequest,
GetDraftPolicyRequest, GetGatewayConfigRequest, GetProviderRequest, GetSandboxConfigRequest,
GetSandboxLogsRequest, GetSandboxPolicyStatusRequest, GetSandboxRequest, HealthRequest,
ListProvidersRequest, ListSandboxPoliciesRequest, ListSandboxesRequest, PolicySource,
PolicyStatus, Provider, RejectDraftChunkRequest, Sandbox, SandboxPhase, SandboxPolicy,
SandboxSpec, SandboxTemplate, SetClusterInferenceRequest, SettingScope, SettingValue,
UpdateConfigRequest, UpdateProviderRequest, WatchSandboxRequest, exec_sandbox_event,
setting_value,
};
use openshell_core::settings::{self, SettingValueKind};
use openshell_core::{ObjectId, ObjectName};
Expand Down Expand Up @@ -1579,6 +1580,7 @@ fn print_failure_diagnosis(diagnosis: &openshell_bootstrap::errors::GatewayFailu

/// Provision or start a gateway (local or remote).
#[allow(clippy::too_many_arguments)] // user-facing CLI command
#[allow(clippy::fn_params_excessive_bools)]
pub async fn gateway_admin_deploy(
name: &str,
remote: Option<&str>,
Expand All @@ -1588,6 +1590,7 @@ pub async fn gateway_admin_deploy(
recreate: bool,
disable_tls: bool,
disable_gateway_auth: bool,
service_base_domains: Vec<String>,
registry_username: Option<&str>,
registry_token: Option<&str>,
gpu: Vec<String>,
Expand Down Expand Up @@ -1649,6 +1652,7 @@ pub async fn gateway_admin_deploy(
.with_port(effective_port)
.with_disable_tls(disable_tls)
.with_disable_gateway_auth(disable_gateway_auth)
.with_service_base_domains(service_base_domains)
.with_gpu(gpu)
.with_recreate(recreate);
if let Some(opts) = remote_opts {
Expand Down Expand Up @@ -3712,6 +3716,57 @@ fn parse_credential_pairs(items: &[String]) -> Result<HashMap<String, String>> {
Ok(map)
}

pub async fn service_expose(
server: &str,
sandbox: &str,
service: &str,
target_port: u16,
tls: &TlsOptions,
) -> Result<()> {
let mut client = grpc_client(server, tls).await?;
let response = client
.expose_service(ExposeServiceRequest {
sandbox: sandbox.to_string(),
service: service.to_string(),
target_port: u32::from(target_port),
domain: true,
})
.await
.map_err(|status| miette::miette!("expose service failed: {status}"))?
.into_inner();

println!(
"{} Exposed service {} on sandbox {} -> 127.0.0.1:{}",
"✓".green().bold(),
service.bold(),
sandbox.bold(),
target_port,
);
if !response.url.is_empty() {
let url = service_url_for_gateway(&response.url, server);
println!(" URL: {}", url.cyan());
}
Ok(())
}

fn service_url_for_gateway(service_url: &str, gateway_endpoint: &str) -> String {
let (Ok(mut service_url), Ok(gateway_endpoint)) = (
url::Url::parse(service_url),
url::Url::parse(gateway_endpoint),
) else {
return service_url.to_string();
};

if service_url.set_scheme(gateway_endpoint.scheme()).is_err() {
return service_url.to_string();
}
if service_url.set_port(gateway_endpoint.port()).is_err() {
return service_url.to_string();
}

service_url.to_string()
}

pub async fn provider_create(
server: &str,
name: &str,
Expand Down Expand Up @@ -5717,8 +5772,8 @@ mod tests {
gateway_type_label, git_sync_files, http_health_check, image_requests_gpu,
inferred_provider_type, parse_cli_setting_value, parse_credential_pairs,
plaintext_gateway_is_remote, provisioning_timeout_message, ready_false_condition_message,
resolve_gateway_control_target_from, sandbox_should_persist, shell_escape,
source_requests_gpu, validate_gateway_name, validate_ssh_host,
resolve_gateway_control_target_from, sandbox_should_persist, service_url_for_gateway,
shell_escape, source_requests_gpu, validate_gateway_name, validate_ssh_host,
};
use crate::TEST_ENV_LOCK;
use hyper::StatusCode;
Expand Down Expand Up @@ -5964,6 +6019,28 @@ mod tests {
assert!(!source_requests_gpu("base"));
}

#[test]
fn service_url_for_gateway_uses_external_gateway_port() {
assert_eq!(
service_url_for_gateway(
"https://quiet-flamingo--openclaw.navigator.openshell.localhost:8080/",
"https://127.0.0.1:31886"
),
"https://quiet-flamingo--openclaw.navigator.openshell.localhost:31886/"
);
}

#[test]
fn service_url_for_gateway_omits_default_external_port() {
assert_eq!(
service_url_for_gateway(
"http://quiet-flamingo--openclaw.navigator.openshell.localhost:8080/",
"https://gateway.example.com"
),
"https://quiet-flamingo--openclaw.navigator.openshell.localhost/"
);
}

#[test]
fn ready_false_condition_message_prefers_reason_and_message() {
let status = SandboxStatus {
Expand Down
9 changes: 9 additions & 0 deletions crates/openshell-cli/tests/ensure_providers_integration.rs
Original file line number Diff line number Diff line change
Expand Up @@ -190,6 +190,15 @@ impl OpenShell for TestOpenShell {
Ok(Response::new(CreateSshSessionResponse::default()))
}

async fn expose_service(
&self,
_request: tonic::Request<openshell_core::proto::ExposeServiceRequest>,
) -> Result<Response<openshell_core::proto::ServiceEndpointResponse>, Status> {
Ok(Response::new(
openshell_core::proto::ServiceEndpointResponse::default(),
))
}

async fn revoke_ssh_session(
&self,
_request: tonic::Request<RevokeSshSessionRequest>,
Expand Down
Loading
Loading