From d86cc83d2dc7637a7e17daaf47af1bd0e0522296 Mon Sep 17 00:00:00 2001 From: John Myers <9696606+johntmyers@users.noreply.github.com> Date: Wed, 12 Aug 2026 13:18:45 -0700 Subject: [PATCH 01/30] feat(network): enable Docker transparent TCP egress Signed-off-by: John Myers <9696606+johntmyers@users.noreply.github.com> --- .../skills/debug-openshell-cluster/SKILL.md | 1 + .../skills/generate-sandbox-policy/SKILL.md | 17 +- .agents/skills/openshell-cli/SKILL.md | 5 + Cargo.lock | 2 + architecture/compute-runtimes.md | 12 +- architecture/sandbox.md | 33 +- crates/openshell-core/src/sandbox_env.rs | 8 + crates/openshell-driver-docker/README.md | 1 + crates/openshell-driver-docker/src/lib.rs | 4 + crates/openshell-driver-docker/src/tests.rs | 21 + .../openshell-driver-kubernetes/src/driver.rs | 30 ++ .../openshell-driver-podman/src/container.rs | 26 ++ crates/openshell-driver-vm/src/driver.rs | 36 ++ crates/openshell-sandbox/src/lib.rs | 111 +++++- .../data/sandbox-policy.rego | 9 +- .../openshell-supervisor-network/src/opa.rs | 7 + .../src/policy_dns/mod.rs | 11 +- .../src/policy_dns/runtime.rs | 216 ++++++++++ .../src/policy_dns/wire.rs | 38 +- .../openshell-supervisor-network/src/proxy.rs | 374 +++++++++++++++++- .../src/proxy/egress.rs | 6 +- .../openshell-supervisor-network/src/run.rs | 178 ++++++++- .../openshell-supervisor-process/Cargo.toml | 2 + .../src/netns/mod.rs | 235 ++++++++++- .../src/netns/nft_ruleset.rs | 183 +++++++++ .../src/process.rs | 1 + 26 files changed, 1515 insertions(+), 52 deletions(-) create mode 100644 crates/openshell-supervisor-network/src/policy_dns/runtime.rs diff --git a/.agents/skills/debug-openshell-cluster/SKILL.md b/.agents/skills/debug-openshell-cluster/SKILL.md index 519275e500..f06d3df470 100644 --- a/.agents/skills/debug-openshell-cluster/SKILL.md +++ b/.agents/skills/debug-openshell-cluster/SKILL.md @@ -186,6 +186,7 @@ Common findings: callbacks. On an older release, set `bind_address = "127.0.0.1:17670"` or upgrade. - Supervisor image exits before printing `openshell-sandbox --version`: the image should be the scratch supervisor image from `deploy/docker/Dockerfile.supervisor` and must contain a static executable at `/openshell-sandbox`. +- A sandbox with explicit `protocol: tcp` endpoints fails before workload readiness: confirm the Docker driver supplied the `policy-dns-transparent-tcp` runtime capability and inspect supervisor logs for missing `nft`, synthetic-route overlap, or namespace-local DNS/TCP listener bind failures. Podman, Kubernetes, VM, sidecar, and out-of-tree drivers must reject this policy until they provide the complete substrate; use omitted protocol with an explicit proxy on those runtimes. - `mise run e2e:docker:gpu` fails with `docker info --format json did not report any discovered NVIDIA CDI GPU devices`: Docker may report `CDISpecDirs` while still having no generated NVIDIA CDI specs. Verify `.DiscoveredDevices` contains entries such as `nvidia.com/gpu=all`, verify `/etc/cdi` or `/var/run/cdi` contains a generated NVIDIA spec, and check that `nvidia-cdi-refresh.service` and `nvidia-cdi-refresh.path` from NVIDIA Container Toolkit are enabled and healthy. The service is a one-shot unit, so `inactive (dead)` can be normal after a successful run; use `systemctl status` and `journalctl` to distinguish success from a skipped or failed refresh. NVIDIA recommends enabling the path and service units, and restarting `nvidia-cdi-refresh.service` to regenerate missing or stale CDI specs. If specs are generated but Docker still reports no discovered devices, restart Docker or reload the daemon and re-check `docker info`. For source checkout development, restart the local gateway with: diff --git a/.agents/skills/generate-sandbox-policy/SKILL.md b/.agents/skills/generate-sandbox-policy/SKILL.md index f6a540d3de..734e188783 100644 --- a/.agents/skills/generate-sandbox-policy/SKILL.md +++ b/.agents/skills/generate-sandbox-policy/SKILL.md @@ -42,11 +42,11 @@ For this tier, default to: - `access: read-only` when the user says "read", "browse", "view", "query", "fetch" - `access: read-write` when the user says "read-write", "create", "update" (but not "delete") - `access: full` when the user says "full access", "everything", "unrestricted" -- L4-only (omit `protocol`, or use explicit `protocol: tcp`) when the user says - "just allow it", "pass through", "no inspection". Prefer omission unless the - user wants the transport intent stated explicitly. Explicit TCP requires a - valid DNS hostname; omit `protocol` for a legacy hostless `allowed_ips` - proxy endpoint. +- L4-only when the user says "just allow it", "pass through", or "no + inspection". Omit `protocol` for explicit-proxy clients. Use + `protocol: tcp` only when the workload must use native DNS and direct socket + calls, the endpoint has a valid DNS hostname, and the selected runtime + advertises policy DNS and transparent TCP support. ### Moderate Tier (host + partial path knowledge) @@ -192,7 +192,8 @@ Follow this decision tree based on the detail tier and user intent: ``` Is L7 inspection needed? ├─ No (user wants pass-through / "just allow it") -│ └─ Generate L4-only policy (no protocol, or protocol: tcp with a DNS hostname; no tls/rules/access) +│ ├─ Explicit-proxy client → omit protocol +│ └─ Native DNS/socket client with a DNS hostname on a supported runtime → protocol: tcp │ └─ Yes (user wants method/path control) │ @@ -213,7 +214,7 @@ Is L7 inspection needed? | API host port | TLS setting | |--------------|-------------| | Port 443 (HTTPS) and L7 rules/preset needed | `tls: terminate` (required for inspection) | -| Port 443 (HTTPS) and L4-only | Omit `tls` (passthrough, no L7) | +| Port 443 (HTTPS) and L4-only | Omit `tls` (passthrough, no L7); choose omitted protocol or explicit TCP based on client/runtime as above | | Non-443 (HTTP) | Omit `tls` | **Critical**: `protocol: rest` on port 443 without `tls: terminate` will not work — the proxy cannot inspect encrypted traffic. Always set `tls: terminate` when combining port 443 with L7 rules. @@ -415,7 +416,7 @@ Evaluate the generated policy for overly broad access and **include warnings in | Condition | Warning to show | |-----------|----------------| -| **L4-only** (no `protocol`, or `protocol: tcp`) | "This policy allows all HTTP methods and paths without inspection. The proxy will only check host:port and binary identity. Consider adding `protocol: rest` with a preset if you want method-level control." | +| **L4-only** (no `protocol`, or `protocol: tcp`) | "This policy allows all application methods and paths without inspection. An omitted protocol uses explicit-proxy behavior; `protocol: tcp` enables policy DNS and transparent TCP only on a runtime that advertises the complete substrate (currently Docker). Consider `protocol: rest` with a preset if you want HTTP method-level control." | | **`access: full`** | "This policy allows all HTTP methods (including DELETE) on all paths. If you don't need DELETE, `read-write` is safer. If you only need to read, `read-only` is the most restrictive option." | | **`access: full` + `enforcement: audit`** | "Full access in audit mode provides no actual restriction — all traffic flows through. This is effectively a monitoring-only policy." | | **`access: read-write`** when user hasn't confirmed write need | "This policy allows POST, PUT, and PATCH on all paths. If you only need to read data, `read-only` is more restrictive." | diff --git a/.agents/skills/openshell-cli/SKILL.md b/.agents/skills/openshell-cli/SKILL.md index 8701c7a40d..7e82488d06 100644 --- a/.agents/skills/openshell-cli/SKILL.md +++ b/.agents/skills/openshell-cli/SKILL.md @@ -335,6 +335,11 @@ This is the most important multi-step workflow. It enables a tight feedback cycl **Key concept**: Policies have static fields (immutable after creation: `filesystem_policy`, `landlock`, `process`) and two dynamic fields: `network_policies` and `network_middlewares`. Both dynamic fields can be updated without recreating the sandbox. +An endpoint with omitted `protocol` retains explicit-proxy behavior. Explicit +`protocol: tcp` requests policy DNS and transparent TCP and currently requires +the Docker runtime; unsupported runtimes reject the policy before starting the +workload rather than activating only part of the network contract. + ``` Create sandbox with initial policy │ diff --git a/Cargo.lock b/Cargo.lock index e3a3fe2d34..be33e3ccd1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4338,6 +4338,7 @@ dependencies = [ "base64 0.22.1", "capctl", "hex", + "ipnet", "landlock", "libc", "miette", @@ -4351,6 +4352,7 @@ dependencies = [ "seccompiler", "serde_json", "sha2 0.10.9", + "socket2 0.6.3", "tempfile", "tokio", "tokio-stream", diff --git a/architecture/compute-runtimes.md b/architecture/compute-runtimes.md index 7da6d4c370..f74b636120 100644 --- a/architecture/compute-runtimes.md +++ b/architecture/compute-runtimes.md @@ -139,7 +139,7 @@ delete, reconciliation removes the row; otherwise it can remain `Deleting`. | Runtime | Best fit | Sandbox boundary | Notes | |---|---|---|---| -| Docker | Local development with Docker available. | Container plus nested sandbox namespace. | Uses host networking so loopback gateway endpoints work from the supervisor. | +| Docker | Local development with Docker available. | Container plus nested sandbox namespace. | Uses host networking so loopback gateway endpoints work from the supervisor. Advertises the combined-supervisor policy-DNS and transparent-TCP substrate. | | Podman | Rootless or single-machine deployments. | Container plus nested sandbox namespace. | Uses the Podman REST API and CDI GPU devices when available. Delivers the supervisor via OCI image volume by default; falls back to extracting the binary to a host-side cache and bind-mounting it when `userns` is configured (overlay does not support idmapped mounts). | | Kubernetes | Cluster deployment through Helm. | Pod plus nested sandbox namespace. | Uses Kubernetes API objects, service accounts, secrets, PVC-backed workspace storage, and GPU resources. | | VM | Experimental microVM isolation. | Per-sandbox libkrun VM. | Managed endpoint-backed driver. The gateway spawns `openshell-driver-vm`, waits for its Unix socket, and then consumes it through the same remote `compute_driver.proto` path used by unmanaged endpoint drivers. The VM driver boots a cached bootstrap `rootfs.ext4`, prepares requested OCI images inside a bootstrap VM with `umoci`, attaches the prepared image disk read-only, and gives each sandbox a writable `overlay.ext4` for merged-root changes and runtime material. The driver persists each accepted launch request beside the overlay and restarts those VMs on driver startup without recreating the overlay. | @@ -160,6 +160,16 @@ operator override because they place gateway-host filesystem state inside the sandbox and can negate OpenShell workspace isolation and filesystem-policy controls. Driver-owned supervisor, token, and TLS bind mounts stay reserved. +Network features follow the existing driver/substrate split. Compute drivers +advertise only the runtime mechanics they can guarantee: namespace and +capability ownership, DNS/TCP capture installation, and coupled +restart ordering. The shared supervisor remains the sole owner of DNS +eligibility, synthetic mappings, process authorization, destination filtering, +pinned dialing, relay behavior, and OCSF decisions. Docker currently advertises +`policy-dns-transparent-tcp`; other runtimes reject explicit TCP policy until +they implement and validate the same complete contract. The capability marker +is driver-owned supervisor input and is removed from workload environments. + Kubernetes deployments may set an AppArmor profile on sandbox agent containers through the driver configuration. The Helm chart defaults sandbox agents to `Unconfined` so runtime/default AppArmor profiles do not block supervisor diff --git a/architecture/sandbox.md b/architecture/sandbox.md index db55a7fa52..333410623b 100644 --- a/architecture/sandbox.md +++ b/architecture/sandbox.md @@ -74,19 +74,26 @@ the shared raw byte relay after the existing adapter gates. Forward HTTP retains its guarded single-request relay while sharing authorization, request context, policy-pinning, and destination boundaries. Adapter-specific response and OCSF event shapes remain at the protocol boundary. -Policy authors may use `protocol: tcp` as an explicit spelling of the existing -L4 passthrough behavior. Explicit TCP endpoints require a valid DNS hostname; -hostless `allowed_ips` and literal-IP selectors remain available only to the -legacy forward-proxy path when `protocol` is omitted. The network supervisor -contains a dormant policy-DNS boundary for explicit TCP endpoints: -it snapshots eligible endpoint identities from one policy generation, resolves -eligible names only through an explicitly supplied trusted resolver, filters -answers through the shared destination controls, and publishes expiring -synthetic-address mappings with separate mapping generations. Refreshes retain -their synthetic identity, and policy reload, expiry, wrong ports, missing -mappings, or pool exhaustion fail closed. The pinned connector never resolves -the name again. No DNS listener is exposed to workloads, resolver configuration -is not injected, and transparent TCP capture is not active in this increment. +An explicit `protocol: tcp` endpoint with a valid DNS hostname opts into native +DNS and transparent TCP when the selected runtime advertises that substrate. +Hostless `allowed_ips` and literal-IP selectors remain available only to the +legacy explicit-proxy path when `protocol` is omitted. The shared supervisor +answers only eligible DNS names, returns an epoch-scoped synthetic address, and +publishes the expiring name, endpoint, ports, policy generation, and validated +real addresses as one correlation. A connection to that synthetic address is +captured before the bypass fence, mapped back to its workload process, authorized +through the same egress pipeline, and dialed only through the pinned addresses. +Omitted protocol endpoints retain explicit-proxy behavior. + +The DNS store is in-memory and sandbox-local. A combined-supervisor restart also +restarts its workload; before execution, the supervisor advances a persisted +boot epoch and installs only that epoch's synthetic capture ranges. An address +cached from the preceding epoch therefore falls through to the bypass fence +instead of inheriting a new mapping. Policy reload, expiry, wrong ports, direct real-IP access, missing +mappings, or pool exhaustion fail closed. Resolver injection, DNS listeners, +capture rules, and the transparent listener are all ready before workload +execution. A runtime that cannot provide the complete contract rejects a policy +containing explicit TCP endpoints rather than partially activating it. Provider credential placeholders are resolved through the live provider state for each HTTP request, after destination and L7 policy admission. A static diff --git a/crates/openshell-core/src/sandbox_env.rs b/crates/openshell-core/src/sandbox_env.rs index 1549258fa3..c135524b70 100644 --- a/crates/openshell-core/src/sandbox_env.rs +++ b/crates/openshell-core/src/sandbox_env.rs @@ -36,6 +36,14 @@ pub const SUPERVISOR_TOPOLOGY: &str = "OPENSHELL_SUPERVISOR_TOPOLOGY"; /// Network enforcement backend selected by the compute driver. pub const NETWORK_ENFORCEMENT_MODE: &str = "OPENSHELL_NETWORK_ENFORCEMENT_MODE"; +/// Comma-separated runtime networking capabilities supplied by the compute +/// driver. Capabilities describe substrate the shared supervisor may activate; +/// they never move policy evaluation into the driver. +pub const NETWORK_RUNTIME_CAPABILITIES: &str = "OPENSHELL_NETWORK_RUNTIME_CAPABILITIES"; + +/// Driver capability for policy-gated DNS and transparent TCP interception. +pub const POLICY_DNS_TRANSPARENT_TCP_CAPABILITY: &str = "policy-dns-transparent-tcp"; + /// Whether network policy evaluation must bind requests to the peer binary. /// /// The default when unset is `"required"`. Kubernetes sidecar experiments may diff --git a/crates/openshell-driver-docker/README.md b/crates/openshell-driver-docker/README.md index 48e56de7bc..e5e9f9c1d2 100644 --- a/crates/openshell-driver-docker/README.md +++ b/crates/openshell-driver-docker/README.md @@ -86,6 +86,7 @@ contract: | `restart_policy = unless-stopped` | Keeps managed sandboxes resumable across daemon or gateway restarts. | | `PidsLimit` | Enforces the sandbox PID budget at the Docker cgroup layer. Set `[openshell.drivers.docker].sandbox_pids_limit = 0` to inherit the Docker/runtime default. | | CDI GPU request | Uses opaque `driver_config.cdi_devices` values when set; otherwise selects the requested count of NVIDIA CDI GPUs in round-robin order when daemon CDI support is detected. Docker daemon `/info` can permit `nvidia.com/gpu=all` as a WSL2 all-only compatibility fallback, where it counts as one selectable device. Exact CDI device lists must not contain duplicates and must match the effective GPU count. | +| `policy-dns-transparent-tcp` capability | Declares that the combined Docker supervisor can own namespace-local DNS/TCP capture and coupled workload restart. The shared supervisor still owns DNS eligibility, mappings, authorization, pinned dialing, relaying, and OCSF decisions. The marker is stripped from the workload environment. | The agent child process does not retain these supervisor privileges. diff --git a/crates/openshell-driver-docker/src/lib.rs b/crates/openshell-driver-docker/src/lib.rs index b1fb5ec224..dcc8c263d9 100644 --- a/crates/openshell-driver-docker/src/lib.rs +++ b/crates/openshell-driver-docker/src/lib.rs @@ -2456,6 +2456,10 @@ fn build_environment_for_oci_user( openshell_core::sandbox_env::TELEMETRY_ENABLED.to_string(), openshell_core::telemetry::enabled_env_value().to_string(), ); + environment.insert( + openshell_core::sandbox_env::NETWORK_RUNTIME_CAPABILITIES.to_string(), + openshell_core::sandbox_env::POLICY_DNS_TRANSPARENT_TCP_CAPABILITY.to_string(), + ); // The root supervisor executes namespace helpers during bootstrap; keep // their search path driver-owned even when the template/spec set PATH. environment.insert("PATH".to_string(), SUPERVISOR_PATH.to_string()); diff --git a/crates/openshell-driver-docker/src/tests.rs b/crates/openshell-driver-docker/src/tests.rs index eddfd778bd..cfcdc32d00 100644 --- a/crates/openshell-driver-docker/src/tests.rs +++ b/crates/openshell-driver-docker/src/tests.rs @@ -619,6 +619,27 @@ fn build_environment_sets_docker_tls_paths() { assert!(env.contains(&"TEMPLATE_ENV=template".to_string())); assert!(env.contains(&"SPEC_ENV=spec".to_string())); assert!(env.contains(&"OPENSHELL_SANDBOX_COMMAND=sleep infinity".to_string())); + assert!(env.contains(&format!( + "{}={}", + openshell_core::sandbox_env::NETWORK_RUNTIME_CAPABILITIES, + openshell_core::sandbox_env::POLICY_DNS_TRANSPARENT_TCP_CAPABILITY + ))); +} + +#[test] +fn build_environment_keeps_network_capabilities_driver_controlled() { + let mut sandbox = test_sandbox(); + sandbox.spec.as_mut().unwrap().environment.insert( + openshell_core::sandbox_env::NETWORK_RUNTIME_CAPABILITIES.to_string(), + "spoofed".to_string(), + ); + let env = build_environment(&sandbox, &runtime_config()); + assert!(env.contains(&format!( + "{}={}", + openshell_core::sandbox_env::NETWORK_RUNTIME_CAPABILITIES, + openshell_core::sandbox_env::POLICY_DNS_TRANSPARENT_TCP_CAPABILITY + ))); + assert!(!env.iter().any(|entry| entry.ends_with("=spoofed"))); } #[test] diff --git a/crates/openshell-driver-kubernetes/src/driver.rs b/crates/openshell-driver-kubernetes/src/driver.rs index ddc7fe2a4f..2fd163780f 100644 --- a/crates/openshell-driver-kubernetes/src/driver.rs +++ b/crates/openshell-driver-kubernetes/src/driver.rs @@ -3994,6 +3994,13 @@ fn apply_required_env( openshell_core::sandbox_env::TELEMETRY_ENABLED, openshell_core::telemetry::enabled_env_value(), ); + // Runtime capabilities are driver-owned. Kubernetes topologies do not yet + // provide the complete policy DNS and transparent TCP substrate. + upsert_env( + env, + openshell_core::sandbox_env::NETWORK_RUNTIME_CAPABILITIES, + "", + ); if !ssh_socket_path.is_empty() { upsert_env( env, @@ -7148,6 +7155,29 @@ mod tests { ); } + #[test] + fn sandbox_pod_clears_unsupported_network_capabilities() { + let spec = SandboxSpec { + environment: std::collections::HashMap::from([( + openshell_core::sandbox_env::NETWORK_RUNTIME_CAPABILITIES.to_string(), + openshell_core::sandbox_env::POLICY_DNS_TRANSPARENT_TCP_CAPABILITY.to_string(), + )]), + ..SandboxSpec::default() + }; + let cr = sandbox_to_k8s_spec_for_test(Some(&spec), &SandboxPodParams::default()); + let env = cr["spec"]["podTemplate"]["spec"]["containers"][0]["env"] + .as_array() + .unwrap(); + let entries = env + .iter() + .filter(|entry| { + entry["name"] == openshell_core::sandbox_env::NETWORK_RUNTIME_CAPABILITIES + }) + .collect::>(); + assert_eq!(entries.len(), 1); + assert_eq!(entries[0]["value"], serde_json::json!("")); + } + #[test] fn node_selector_from_platform_config() { let template = SandboxTemplate { diff --git a/crates/openshell-driver-podman/src/container.rs b/crates/openshell-driver-podman/src/container.rs index cae477618c..9f269c6e29 100644 --- a/crates/openshell-driver-podman/src/container.rs +++ b/crates/openshell-driver-podman/src/container.rs @@ -524,6 +524,12 @@ fn build_env( openshell_core::sandbox_env::TELEMETRY_ENABLED.into(), openshell_core::telemetry::enabled_env_value().into(), ); + // Runtime capabilities are driver-owned. Podman does not yet provide the + // policy DNS and transparent TCP substrate, so override image/user input. + env.insert( + openshell_core::sandbox_env::NETWORK_RUNTIME_CAPABILITIES.into(), + String::new(), + ); // 3. TLS client cert paths (when mTLS is enabled). These point to // the container-side mount paths where the cert files are @@ -1969,6 +1975,26 @@ mod tests { ); } + #[test] + fn container_spec_clears_unsupported_network_capabilities() { + use openshell_core::proto::compute::v1::{DriverSandboxSpec, DriverSandboxTemplate}; + + let mut sandbox = test_sandbox("test-id", "legit-name"); + sandbox.spec = Some(DriverSandboxSpec { + environment: std::collections::HashMap::from([( + openshell_core::sandbox_env::NETWORK_RUNTIME_CAPABILITIES.to_string(), + openshell_core::sandbox_env::POLICY_DNS_TRANSPARENT_TCP_CAPABILITY.to_string(), + )]), + template: Some(DriverSandboxTemplate::default()), + ..Default::default() + }); + let spec = build_container_spec(&sandbox, &test_config()); + assert_eq!( + spec["env"][openshell_core::sandbox_env::NETWORK_RUNTIME_CAPABILITIES], + serde_json::json!("") + ); + } + /// Extract the container spec's supervisor argv (`command`) as strings. fn spec_command(spec: &Value) -> Vec { spec["command"] diff --git a/crates/openshell-driver-vm/src/driver.rs b/crates/openshell-driver-vm/src/driver.rs index 982d678065..9b243d95d3 100644 --- a/crates/openshell-driver-vm/src/driver.rs +++ b/crates/openshell-driver-vm/src/driver.rs @@ -4461,6 +4461,12 @@ fn build_guest_environment( openshell_core::sandbox_env::TELEMETRY_ENABLED.to_string(), openshell_core::telemetry::enabled_env_value().to_string(), ); + // Runtime capabilities are driver-owned. The VM driver does not yet + // provide policy DNS and transparent TCP interception. + environment.insert( + openshell_core::sandbox_env::NETWORK_RUNTIME_CAPABILITIES.to_string(), + String::new(), + ); environment.remove(openshell_core::sandbox_env::SANDBOX_TOKEN); environment.remove(openshell_core::sandbox_env::SANDBOX_TOKEN_FILE); // Prevent user-supplied environment from overriding the TLS server name @@ -7068,6 +7074,36 @@ mod tests { ); } + #[test] + fn build_guest_environment_clears_unsupported_network_capabilities() { + let config = VmDriverConfig { + openshell_endpoint: "http://127.0.0.1:8080".to_string(), + ..Default::default() + }; + let sandbox = Sandbox { + id: "sandbox-123".to_string(), + name: "sandbox-123".to_string(), + spec: Some(SandboxSpec { + environment: HashMap::from([( + openshell_core::sandbox_env::NETWORK_RUNTIME_CAPABILITIES.to_string(), + openshell_core::sandbox_env::POLICY_DNS_TRANSPARENT_TCP_CAPABILITY.to_string(), + )]), + ..Default::default() + }), + ..Default::default() + }; + let env = build_guest_environment(&sandbox, &config, None); + assert!(env.contains(&format!( + "{}=", + openshell_core::sandbox_env::NETWORK_RUNTIME_CAPABILITIES + ))); + assert!(!env.contains(&format!( + "{}={}", + openshell_core::sandbox_env::NETWORK_RUNTIME_CAPABILITIES, + openshell_core::sandbox_env::POLICY_DNS_TRANSPARENT_TCP_CAPABILITY + ))); + } + #[test] fn build_guest_environment_uses_endpoint_override_for_tap() { let config = VmDriverConfig { diff --git a/crates/openshell-sandbox/src/lib.rs b/crates/openshell-sandbox/src/lib.rs index 56a380b19c..43457a5bbb 100644 --- a/crates/openshell-sandbox/src/lib.rs +++ b/crates/openshell-sandbox/src/lib.rs @@ -73,6 +73,15 @@ const SIDECAR_NETWORK_ENFORCEMENT_MODE: &str = "sidecar-nftables"; const SIDECAR_TLS_DIR: &str = openshell_core::container_paths::SIDECAR_TLS_DIR; const SIDECAR_CA_CERT: &str = "openshell-ca.pem"; const SIDECAR_CA_BUNDLE: &str = "ca-bundle.pem"; + +#[cfg(any(test, target_os = "linux"))] +fn has_network_runtime_capability(capabilities: Option<&str>, required: &str) -> bool { + capabilities.is_some_and(|capabilities| { + capabilities + .split(',') + .any(|capability| capability.trim() == required) + }) +} const SIDECAR_PROCESS_PROXY_ADDR: &str = "127.0.0.1:3128"; const SIDECAR_READY_TIMEOUT_SECS: u64 = 120; @@ -374,6 +383,81 @@ pub async fn run_sandbox( None }; + #[cfg(target_os = "linux")] + let transparent_tcp_requested = opa_engine + .as_ref() + .map(|engine| engine.policy_dns_eligibility_snapshot()) + .transpose()? + .is_some_and(|snapshot| !snapshot.endpoints.is_empty()); + #[cfg(target_os = "linux")] + let runtime_capabilities = + std::env::var(openshell_core::sandbox_env::NETWORK_RUNTIME_CAPABILITIES).ok(); + #[cfg(target_os = "linux")] + let transparent_tcp_capable = has_network_runtime_capability( + runtime_capabilities.as_deref(), + openshell_core::sandbox_env::POLICY_DNS_TRANSPARENT_TCP_CAPABILITY, + ); + #[cfg(target_os = "linux")] + let transparent_runtime = if transparent_tcp_requested { + if !transparent_tcp_capable { + ocsf_emit!( + ConfigStateChangeBuilder::new(ocsf_ctx()) + .severity(SeverityId::Medium) + .status(StatusId::Failure) + .state(StateId::Disabled, "unsupported_runtime") + .message( + "Policy DNS and transparent TCP unavailable: runtime capability is missing" + ) + .build() + ); + return Err(miette::miette!( + "policy contains protocol: tcp endpoints, but the selected runtime does not advertise policy DNS and transparent TCP support" + )); + } + if sidecar_network_enforcement { + ocsf_emit!( + ConfigStateChangeBuilder::new(ocsf_ctx()) + .severity(SeverityId::Medium) + .status(StatusId::Failure) + .state(StateId::Disabled, "unsupported_topology") + .message("Policy DNS and transparent TCP unavailable: sidecar topology is unsupported") + .build() + ); + return Err(miette::miette!( + "policy DNS and transparent TCP are not yet supported by the sidecar topology" + )); + } + let namespace = netns.as_ref().ok_or_else(|| { + miette::miette!("policy DNS and transparent TCP require a workload network namespace") + })?; + let listeners = namespace + .bind_transparent_tcp_listeners() + .await + .into_diagnostic() + .wrap_err("failed to bind transparent TCP listeners")?; + let (dns_udp, dns_tcp) = namespace + .bind_policy_dns_sockets() + .await + .into_diagnostic() + .wrap_err("failed to bind policy DNS listeners")?; + let proxy_port = policy + .network + .proxy + .as_ref() + .and_then(|proxy| proxy.http_addr) + .map_or(3128, |address| address.port()); + let runtime = openshell_supervisor_network::run::TransparentRuntimeSetup::new( + listeners, + dns_udp, + dns_tcp, + sandbox_id.as_deref(), + )?; + let (ipv4_cidr, ipv6_cidr) = runtime.synthetic_cidrs(); + namespace.install_transparent_tcp_rules(proxy_port, &ipv4_cidr, &ipv6_cidr)?; + Some(runtime) + } else { + None + }; // The denial channel is owned by the orchestrator: the proxy (in the // networking leaf) and the bypass monitor (in the process leaf) both // produce DenialEvents that the denial aggregator (orchestrator-side) @@ -439,6 +523,8 @@ pub async fn run_sandbox( agent_proposals.clone(), workspace_rx.clone(), &upstream_proxy_args, + #[cfg(target_os = "linux")] + transparent_runtime, ) .await?, ) @@ -1537,6 +1623,7 @@ fn enrich_sandbox_baseline_paths(policy: &mut SandboxPolicy) { mod baseline_tests { use super::*; use openshell_core::policy::{FilesystemPolicy, LandlockPolicy, ProcessPolicy}; + use std::path::PathBuf; #[test] fn proc_not_in_both_read_only_and_read_write_when_gpu_present() { @@ -1768,7 +1855,7 @@ mod baseline_tests { let mut policy = SandboxPolicy { version: 1, filesystem: FilesystemPolicy { - read_only: vec![std::path::PathBuf::from("/tmp")], + read_only: vec![PathBuf::from("/tmp")], read_write: vec![], include_workdir: false, }, @@ -1783,17 +1870,14 @@ mod baseline_tests { enrich_sandbox_baseline_paths(&mut policy); assert!( - policy - .filesystem - .read_only - .contains(&std::path::PathBuf::from("/tmp")), + policy.filesystem.read_only.contains(&PathBuf::from("/tmp")), "explicit read_only baseline path should be preserved" ); assert!( !policy .filesystem .read_write - .contains(&std::path::PathBuf::from("/tmp")), + .contains(&PathBuf::from("/tmp")), "baseline enrichment must not promote explicit read_only /tmp to read_write" ); } @@ -3968,6 +4052,21 @@ fn format_setting_value(es: &openshell_core::proto::EffectiveSetting) -> String )] mod tests { use super::*; + + #[test] + fn transparent_tcp_capability_requires_exact_driver_marker() { + let required = openshell_core::sandbox_env::POLICY_DNS_TRANSPARENT_TCP_CAPABILITY; + assert!(!has_network_runtime_capability(None, required)); + assert!(!has_network_runtime_capability(Some(""), required)); + assert!(!has_network_runtime_capability( + Some("policy-dns-transparent-tcp-extra"), + required + )); + assert!(has_network_runtime_capability( + Some("other, policy-dns-transparent-tcp"), + required + )); + } use openshell_core::policy::{ FilesystemPolicy, LandlockPolicy, NetworkMode, NetworkPolicy, ProcessPolicy, ProxyPolicy, }; diff --git a/crates/openshell-supervisor-network/data/sandbox-policy.rego b/crates/openshell-supervisor-network/data/sandbox-policy.rego index 87469cc6dd..e3fa6d36b4 100644 --- a/crates/openshell-supervisor-network/data/sandbox-policy.rego +++ b/crates/openshell-supervisor-network/data/sandbox-policy.rego @@ -904,8 +904,7 @@ _matching_endpoint_configs := [cfg | # markers needed by later policy-DNS correlation. _policy_endpoint_records(policy_name, policy) := [record | - some endpoint_index - ep := policy.endpoints[endpoint_index] + some endpoint_index, ep in policy.endpoints endpoint_matches_request(ep, input.network) record := { "policy_name": policy_name, @@ -926,10 +925,8 @@ _matching_endpoint_records := [record | # grant access to any process. Only endpoints that explicitly opt into raw TCP # and provide a resolvable host plus concrete ports are materialized. policy_dns_eligible_endpoint_records := [record | - some policy_name - policy := data.network_policies[policy_name] - some endpoint_index - ep := policy.endpoints[endpoint_index] + some policy_name, policy in data.network_policies + some endpoint_index, ep in policy.endpoints lower(object.get(ep, "protocol", "")) == "tcp" object.get(ep, "host", "") != "" ports := object.get(ep, "ports", []) diff --git a/crates/openshell-supervisor-network/src/opa.rs b/crates/openshell-supervisor-network/src/opa.rs index a6d6b777ed..50da116a56 100644 --- a/crates/openshell-supervisor-network/src/opa.rs +++ b/crates/openshell-supervisor-network/src/opa.rs @@ -2219,6 +2219,13 @@ process: assert!(snapshot.endpoints.is_empty()); } + #[test] + fn policy_dns_snapshot_accepts_the_default_multi_policy_shape() { + let engine = OpaEngine::from_strings(TEST_POLICY, TEST_DATA_YAML).unwrap(); + let snapshot = engine.policy_dns_eligibility_snapshot().unwrap(); + assert!(snapshot.endpoints.is_empty()); + } + #[test] fn allowed_binary_and_endpoint() { let engine = test_engine(); diff --git a/crates/openshell-supervisor-network/src/policy_dns/mod.rs b/crates/openshell-supervisor-network/src/policy_dns/mod.rs index e4bd5499b2..11faf7c291 100644 --- a/crates/openshell-supervisor-network/src/policy_dns/mod.rs +++ b/crates/openshell-supervisor-network/src/policy_dns/mod.rs @@ -6,25 +6,26 @@ reason = "the crate-private API is consumed by the runtime activation slice" )] -//! Dormant policy-gated DNS and synthetic resolved-endpoint correlation. +//! Policy-gated DNS and synthetic resolved-endpoint correlation. //! -//! This module implements the DNS security boundary and mapping state only. -//! Runtime listener startup, resolver injection, and transparent TCP capture -//! intentionally land in later stack entries. +//! The shared supervisor owns DNS eligibility and mapping state. Supported +//! runtimes provide namespace-local DNS and transparent TCP capture sockets. #![allow( dead_code, unused_imports, - reason = "PR2 exposes a dormant library boundary consumed by PR3 runtime wiring" + reason = "the policy DNS boundary retains metrics and helpers for later runtime integrations" )] mod name; mod resolver; +mod runtime; mod store; mod wire; pub(crate) use name::NormalizedName; pub(crate) use resolver::{AddressFamily, SocketTrustedResolver, TrustedAnswer, TrustedResolver}; +pub(crate) use runtime::{PolicyDnsRuntime, PolicyDnsRuntimeConfig}; pub(crate) use store::{ MappingLookup, MappingLookupError, PolicyEndpointId, PublishError, PublishRequest, ResolvedEndpointRecord, ResolvedEndpointStore, ResolvedPortContract, StoreConfig, diff --git a/crates/openshell-supervisor-network/src/policy_dns/runtime.rs b/crates/openshell-supervisor-network/src/policy_dns/runtime.rs new file mode 100644 index 0000000000..f65dc4c74e --- /dev/null +++ b/crates/openshell-supervisor-network/src/policy_dns/runtime.rs @@ -0,0 +1,216 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Runtime-owned policy DNS listeners for combined Linux supervisors. + +use super::resolver::MAX_DNS_MESSAGE_BYTES; +use super::store::{ResolvedEndpointStore, StoreConfig, SyntheticPools}; +use super::{PolicyDnsService, SocketTrustedResolver, wire}; +use crate::opa::OpaEngine; +use miette::{IntoDiagnostic, Result, WrapErr}; +use openshell_core::net::set_tcp_nodelay_best_effort; +use openshell_ocsf::{ConfigStateChangeBuilder, SeverityId, StateId, StatusId, ocsf_emit}; +use std::net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr}; +use std::sync::Arc; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; +use tokio::task::JoinHandle; + +const IPV4_POOL_PREFIX: u8 = 25; +const IPV6_POOL_PREFIX: u8 = 120; +const IPV4_EPOCH_WINDOWS: u64 = 1 << (IPV4_POOL_PREFIX - 15); +const MAX_MAPPINGS: usize = 256; + +#[derive(Debug, Clone)] +pub(crate) struct PolicyDnsRuntimeConfig { + pub(crate) ipv4_cidr: ipnet::Ipv4Net, + pub(crate) ipv6_cidr: ipnet::Ipv6Net, + pools: SyntheticPools, +} + +impl PolicyDnsRuntimeConfig { + pub(crate) fn for_epoch(epoch: u64) -> Result { + let ipv4_parent: ipnet::Ipv4Net = "198.18.0.0/15".parse().unwrap(); + let ipv4_window = epoch % IPV4_EPOCH_WINDOWS; + let ipv4_start = u32::from(ipv4_parent.network()) + + u32::try_from(ipv4_window * (1 << (32 - IPV4_POOL_PREFIX))).unwrap(); + let ipv4_cidr = ipnet::Ipv4Net::new(Ipv4Addr::from(ipv4_start), IPV4_POOL_PREFIX) + .map_err(|error| miette::miette!(error.to_string()))?; + + let ipv6_parent: ipnet::Ipv6Net = "fd23:6f70:656e::/48".parse().unwrap(); + let ipv6_window = u128::from(epoch); + let ipv6_start = + u128::from(ipv6_parent.network()) + (ipv6_window << (128 - IPV6_POOL_PREFIX)); + let ipv6_cidr = ipnet::Ipv6Net::new(Ipv6Addr::from(ipv6_start), IPV6_POOL_PREFIX) + .map_err(|error| miette::miette!(error.to_string()))?; + + let pools = SyntheticPools::new( + ipv4_cidr.network()..=ipv4_cidr.broadcast(), + ipv6_cidr.network()..=ipv6_cidr.broadcast(), + ) + .map_err(|error| miette::miette!(error.to_string()))?; + Ok(Self { + ipv4_cidr, + ipv6_cidr, + pools, + }) + } +} + +pub(crate) struct PolicyDnsRuntime { + pub(crate) store: Arc, + tasks: Vec>, +} + +impl PolicyDnsRuntime { + pub(crate) fn start( + policy: Arc, + udp: tokio::net::UdpSocket, + tcp: tokio::net::TcpListener, + trusted_host_gateway: Option, + config: PolicyDnsRuntimeConfig, + engine_ready: tokio::sync::watch::Receiver, + ) -> Result { + let upstream = trusted_resolver_from_resolv_conf()?; + let store = Arc::new(ResolvedEndpointStore::new( + StoreConfig::new(config.pools, MAX_MAPPINGS) + .map_err(|error| miette::miette!(error.to_string()))?, + )); + let service = Arc::new(PolicyDnsService::new( + policy, + SocketTrustedResolver::new(upstream), + store.clone(), + trusted_host_gateway, + )); + let address = udp.local_addr().into_diagnostic()?; + + let udp_service = service.clone(); + let mut udp_engine_ready = engine_ready.clone(); + let udp_task = tokio::spawn(async move { + if udp_engine_ready.wait_for(|ready| *ready).await.is_err() { + return; + } + let mut request = vec![0_u8; MAX_DNS_MESSAGE_BYTES + 1]; + loop { + let Ok((length, peer)) = udp.recv_from(&mut request).await else { + break; + }; + if let Ok(response) = wire::handle_udp_query(&udp_service, &request[..length]).await + { + let _ = udp.send_to(&response, peer).await; + } + } + }); + + let mut tcp_engine_ready = engine_ready; + let tcp_task = tokio::spawn(async move { + if tcp_engine_ready.wait_for(|ready| *ready).await.is_err() { + return; + } + loop { + let Ok((mut stream, _)) = tcp.accept().await else { + break; + }; + set_tcp_nodelay_best_effort(&stream); + let service = service.clone(); + tokio::spawn(async move { + let Ok(wire_length) = stream.read_u16().await else { + return; + }; + let length = usize::from(wire_length); + if length > MAX_DNS_MESSAGE_BYTES { + return; + } + let mut frame = Vec::with_capacity(length + 2); + frame.extend_from_slice(&wire_length.to_be_bytes()); + frame.resize(length + 2, 0); + if stream.read_exact(&mut frame[2..]).await.is_err() { + return; + } + if let Ok(response) = wire::handle_tcp_query(&service, &frame).await { + let _ = stream.write_all(&response).await; + } + }); + } + }); + let expiry_store = store.clone(); + let expiry_task = tokio::spawn(async move { + let mut interval = tokio::time::interval(std::time::Duration::from_secs(1)); + loop { + interval.tick().await; + let _ = expiry_store.expire(std::time::Instant::now()); + } + }); + + ocsf_emit!( + ConfigStateChangeBuilder::new(openshell_ocsf::ctx::ctx()) + .severity(SeverityId::Informational) + .status(StatusId::Success) + .state(StateId::Enabled, "ready") + .message(format!("Policy DNS listening on {address}")) + .build() + ); + Ok(Self { + store, + tasks: vec![udp_task, tcp_task, expiry_task], + }) + } +} + +impl Drop for PolicyDnsRuntime { + fn drop(&mut self) { + for task in &self.tasks { + task.abort(); + } + } +} + +fn trusted_resolver_from_resolv_conf() -> Result { + let contents = std::fs::read_to_string("/etc/resolv.conf") + .into_diagnostic() + .wrap_err("failed to read trusted supervisor resolver configuration")?; + for line in contents.lines() { + let line = line.split('#').next().unwrap_or_default(); + let mut fields = line.split_whitespace(); + if fields.next() != Some("nameserver") { + continue; + } + let Some(value) = fields.next() else { + continue; + }; + if let Ok(ip) = value.parse::() { + return Ok(SocketAddr::new(ip, 53)); + } + } + Err(miette::miette!( + "no literal nameserver is configured in supervisor /etc/resolv.conf" + )) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn production_pool_is_disjoint_from_workload_veth() { + let workload: ipnet::IpNet = "10.200.0.0/24".parse().unwrap(); + let config = PolicyDnsRuntimeConfig::for_epoch(42).unwrap(); + for address in [ + IpAddr::V4(config.ipv4_cidr.network()), + IpAddr::V4(config.ipv4_cidr.broadcast()), + ] { + assert!(!workload.contains(&address)); + } + } + + #[test] + fn adjacent_boot_epochs_use_disjoint_capture_ranges() { + let first = PolicyDnsRuntimeConfig::for_epoch(1).unwrap(); + let second = PolicyDnsRuntimeConfig::for_epoch(2).unwrap(); + assert_ne!(first.ipv4_cidr, second.ipv4_cidr); + assert_ne!(first.ipv6_cidr, second.ipv6_cidr); + let parent: ipnet::Ipv4Net = "198.18.0.0/15".parse().unwrap(); + for address in [first.ipv4_cidr.network(), second.ipv4_cidr.broadcast()] { + assert!(parent.contains(&address)); + } + } +} diff --git a/crates/openshell-supervisor-network/src/policy_dns/wire.rs b/crates/openshell-supervisor-network/src/policy_dns/wire.rs index ecc520b428..d422b95f54 100644 --- a/crates/openshell-supervisor-network/src/policy_dns/wire.rs +++ b/crates/openshell-supervisor-network/src/policy_dns/wire.rs @@ -79,6 +79,9 @@ pub(crate) async fn handle_udp_query( Err(PolicyDnsError::Resolver(ResolveError::NxDomain)) => { encode_message(response_with_code(&request, ResponseCode::NXDomain)) } + Err(PolicyDnsError::Resolver(ResolveError::NoData)) => { + encode_message(response_with_code(&request, ResponseCode::NoError)) + } Err(PolicyDnsError::InvalidName) => { encode_message(response_with_code(&request, ResponseCode::FormErr)) } @@ -162,6 +165,18 @@ mod tests { calls: AtomicUsize, } + struct NoDataResolver; + + impl TrustedResolver for NoDataResolver { + async fn resolve( + &self, + _name: &NormalizedName, + _family: AddressFamily, + ) -> Result { + Err(ResolveError::NoData) + } + } + impl TrustedResolver for FakeResolver { async fn resolve( &self, @@ -179,7 +194,7 @@ mod tests { } } - fn service() -> PolicyDnsService { + fn service_with_resolver(resolver: R) -> PolicyDnsService { let yaml = r" network_policies: database: @@ -200,9 +215,7 @@ process: { run_as_user: sandbox, run_as_group: sandbox } .unwrap(); PolicyDnsService::new( policy, - FakeResolver { - calls: AtomicUsize::new(0), - }, + resolver, Arc::new(ResolvedEndpointStore::new( StoreConfig::new(pools, 8).unwrap(), )), @@ -210,6 +223,12 @@ process: { run_as_user: sandbox, run_as_group: sandbox } ) } + fn service() -> PolicyDnsService { + service_with_resolver(FakeResolver { + calls: AtomicUsize::new(0), + }) + } + fn request(name: &str, record_type: RecordType) -> Vec { let mut message = Message::new(42, MessageType::Query, OpCode::Query); message.metadata.recursion_desired = true; @@ -263,6 +282,17 @@ process: { run_as_user: sandbox, run_as_group: sandbox } assert_eq!(service.resolver.calls.load(Ordering::SeqCst), 0); } + #[tokio::test] + async fn eligible_family_without_records_returns_empty_success() { + let service = service_with_resolver(NoDataResolver); + let wire = handle_udp_query(&service, &request("db.example.", RecordType::AAAA)) + .await + .unwrap(); + let response = Message::from_vec(&wire).unwrap(); + assert_eq!(response.metadata.response_code, ResponseCode::NoError); + assert!(response.answers.is_empty()); + } + #[tokio::test] async fn unsupported_type_is_not_implemented_and_malformed_tcp_is_rejected() { let service = service(); diff --git a/crates/openshell-supervisor-network/src/proxy.rs b/crates/openshell-supervisor-network/src/proxy.rs index b44ffa603e..5704766496 100644 --- a/crates/openshell-supervisor-network/src/proxy.rs +++ b/crates/openshell-supervisor-network/src/proxy.rs @@ -10,6 +10,8 @@ mod relay; use crate::identity::BinaryIdentityCache; use crate::l7::tls::ProxyTlsState; use crate::opa::{NetworkAction, OpaEngine, PolicyGenerationGuard}; +#[cfg(target_os = "linux")] +use crate::policy_dns::{MappingLookupError, PolicyEndpointId, ResolvedEndpointStore}; use crate::policy_local::{POLICY_LOCAL_HOST, PolicyLocalContext}; use crate::upstream_proxy::{self, UpstreamProxyConfig}; use miette::{IntoDiagnostic, Result}; @@ -27,6 +29,8 @@ use openshell_ocsf::{ HttpActivityBuilder, HttpRequest, NetworkActivityBuilder, Process, SeverityId, StatusId, Url as OcsfUrl, ocsf_emit, }; +#[cfg(target_os = "linux")] +use std::mem::size_of; use std::net::{IpAddr, SocketAddr}; use std::path::PathBuf; use std::sync::Arc; @@ -438,6 +442,372 @@ impl Drop for ProxyHandle { } } +/// RAII handle for transparent TCP accept loops. +#[cfg(target_os = "linux")] +pub(crate) struct TransparentTcpHandle { + joins: Vec>, +} + +#[cfg(target_os = "linux")] +impl TransparentTcpHandle { + #[allow(clippy::too_many_arguments)] + pub(crate) fn start( + listeners: Vec, + store: Arc, + opa_engine: Arc, + identity_cache: Arc, + entrypoint_pid: Arc, + agent_proposals: openshell_core::proposals::AgentProposals, + denial_tx: Option>, + activity_tx: Option, + upstream_proxy_args: &upstream_proxy::UpstreamProxyArgs, + engine_ready: tokio::sync::watch::Receiver, + ) -> Result { + let upstream_proxy = Arc::new( + UpstreamProxyConfig::from_args(upstream_proxy_args) + .map_err(|error| miette::miette!(error))?, + ); + let mut joins = Vec::with_capacity(listeners.len()); + for listener in listeners { + let store = store.clone(); + let engine = opa_engine.clone(); + let cache = identity_cache.clone(); + let pid = entrypoint_pid.clone(); + let proposals = agent_proposals.clone(); + let denial_tx = denial_tx.clone(); + let activity_tx = activity_tx.clone(); + let upstream_proxy = upstream_proxy.clone(); + let mut engine_ready = engine_ready.clone(); + joins.push(tokio::spawn(async move { + if tokio::time::timeout( + std::time::Duration::from_secs(15), + engine_ready.wait_for(|ready| *ready), + ) + .await + .is_err() + { + warn!( + "Engine readiness signal not received within 15s; proceeding with transparent TCP accept loop" + ); + } + loop { + let Ok((stream, _)) = listener.accept().await else { + break; + }; + set_tcp_nodelay_best_effort(&stream); + let store = store.clone(); + let engine = engine.clone(); + let cache = cache.clone(); + let pid = pid.clone(); + let proposals = proposals.clone(); + let denial_tx = denial_tx.clone(); + let activity_tx = activity_tx.clone(); + let upstream_proxy = upstream_proxy.clone(); + tokio::spawn(async move { + if let Err(error) = handle_transparent_tcp_connection( + stream, + store, + engine, + cache, + pid, + proposals, + denial_tx, + activity_tx, + upstream_proxy, + ) + .await + { + ocsf_emit!( + NetworkActivityBuilder::new(openshell_ocsf::ctx::ctx()) + .activity(ActivityId::Fail) + .severity(SeverityId::Low) + .status(StatusId::Failure) + .message(format!("Transparent TCP connection error: {error}")) + .build() + ); + } + }); + } + })); + } + Ok(Self { joins }) + } +} + +#[cfg(target_os = "linux")] +impl Drop for TransparentTcpHandle { + fn drop(&mut self) { + for join in &self.joins { + join.abort(); + } + } +} + +#[cfg(target_os = "linux")] +#[allow(clippy::too_many_arguments)] +async fn handle_transparent_tcp_connection( + mut client: TcpStream, + store: Arc, + opa_engine: Arc, + identity_cache: Arc, + entrypoint_pid: Arc, + agent_proposals: openshell_core::proposals::AgentProposals, + denial_tx: Option>, + activity_tx: Option, + upstream_proxy: Arc>, +) -> Result<()> { + let workload_addr = client.peer_addr().into_diagnostic()?; + let original = original_destination(&client).into_diagnostic()?; + let current_generation = opa_engine.current_generation(); + let mapping = match store.lookup( + original.ip(), + original.port(), + current_generation, + std::time::Instant::now(), + ) { + Ok(mapping) => mapping, + Err(error) => { + emit_transparent_mapping_denial(workload_addr, original, error); + emit_activity(&activity_tx, true, "transparent_tcp_mapping"); + return Ok(()); + } + }; + let host = mapping.record.normalized_name.as_str().to_string(); + let port = original.port(); + let Some(pinned_ip) = mapping + .record + .contracts + .iter() + .filter(|contract| contract.port == port) + .flat_map(|contract| contract.pinned_addresses.iter().copied()) + .next() + else { + emit_transparent_mapping_denial( + workload_addr, + original, + MappingLookupError::InvalidMapping, + ); + return Ok(()); + }; + + let connection = crate::procfs::WorkloadProxyTcpConnection::new(workload_addr, original); + let intent = EgressIntent::transparent_tcp(host.clone(), port, pinned_ip); + let engine = opa_engine.clone(); + let cache = identity_cache.clone(); + let pid = entrypoint_pid.clone(); + let decision = tokio::task::spawn_blocking(move || { + authorize_egress_intent(connection, &engine, &cache, &pid, intent) + }) + .await + .map_err(|error| miette::miette!("identity resolution task panicked: {error}"))?; + + if let NetworkAction::Deny { reason } = &decision.action { + emit_transparent_policy_denial(&decision, workload_addr, &host, port); + emit_denial( + &denial_tx, + &host, + port, + decision + .binary + .as_ref() + .map_or("-", |path| path.to_str().unwrap_or("-")), + &decision, + reason, + "transparent-tcp", + ); + emit_activity(&activity_tx, true, "transparent_tcp_policy"); + return Ok(()); + } + + let endpoint_id = decision + .endpoint + .matched_endpoints + .iter() + .map(|endpoint| PolicyEndpointId { + policy_name: endpoint.policy_name.clone(), + endpoint_index: endpoint.endpoint_index, + }) + .find(|candidate| mapping.endpoint_ids().any(|mapped| mapped == candidate)); + let Some(endpoint_id) = endpoint_id else { + let reason = "authorized endpoint did not match DNS correlation"; + emit_transparent_policy_denial(&decision, workload_addr, &host, port); + emit_denial( + &denial_tx, + &host, + port, + decision + .binary + .as_ref() + .map_or("-", |path| path.to_str().unwrap_or("-")), + &decision, + reason, + "transparent-tcp", + ); + emit_activity(&activity_tx, true, "transparent_tcp_policy"); + return Ok(()); + }; + + let connector = mapping.connector_for(&endpoint_id).await.map_err(|error| { + miette::miette!("transparent TCP pinned destination is invalid: {error}") + })?; + let generation_guard = relay::pin_policy_generation(&opa_engine, decision.policy_generation)?; + let mut ctx = relay::http_context( + &decision, + None, + None, + activity_tx.clone(), + None, + agent_proposals, + ); + let middleware_gate = middleware_uninspectable_gate(&opa_engine, &ctx)?; + if middleware_gate == crate::l7::middleware::UninspectableTrafficGate::Deny { + crate::l7::middleware::emit_middleware_uninspectable(&ctx, "transparent tcp", true); + return Ok(()); + } + if middleware_gate == crate::l7::middleware::UninspectableTrafficGate::BypassWithFinding { + crate::l7::middleware::emit_middleware_uninspectable(&ctx, "transparent tcp", false); + } + let mut upstream = dial_upstream(&upstream_proxy, &host, port, connector.addrs()) + .await + .into_diagnostic()?; + generation_guard.ensure_current()?; + ctx.request_default_port = None; + let policy_name = match &decision.action { + NetworkAction::Allow { matched_policy } => matched_policy.as_deref().unwrap_or("-"), + NetworkAction::Deny { .. } => "-", + }; + ocsf_emit!( + NetworkActivityBuilder::new(openshell_ocsf::ctx::ctx()) + .activity(ActivityId::Open) + .action(ActionId::Allowed) + .disposition(DispositionId::Allowed) + .severity(SeverityId::Informational) + .status(StatusId::Success) + .dst_endpoint(Endpoint::from_domain(&host, port)) + .src_endpoint_addr(workload_addr.ip(), workload_addr.port()) + .firewall_rule(policy_name, "opa") + .message(format!("Transparent TCP allowed {host}:{port}")) + .status_detail("transparent_tcp_allowed") + .build() + ); + emit_activity(&activity_tx, false, "transparent_tcp"); + relay::relay_tcp(&mut client, &mut upstream, &generation_guard, &ctx).await +} + +#[cfg(target_os = "linux")] +fn original_destination(stream: &TcpStream) -> std::io::Result { + use std::os::fd::AsRawFd; + let fd = stream.as_raw_fd(); + if stream.local_addr()?.is_ipv4() { + #[allow(unsafe_code)] + unsafe { + let mut address: libc::sockaddr_in = std::mem::zeroed(); + let mut length = size_of::() as libc::socklen_t; + if libc::getsockopt( + fd, + libc::SOL_IP, + 80, // SO_ORIGINAL_DST + std::ptr::addr_of_mut!(address).cast(), + &mut length, + ) != 0 + { + return Err(std::io::Error::last_os_error()); + } + return Ok(SocketAddr::new( + IpAddr::V4(std::net::Ipv4Addr::from( + address.sin_addr.s_addr.to_ne_bytes(), + )), + u16::from_be(address.sin_port), + )); + } + } + #[allow(unsafe_code)] + unsafe { + let mut address: libc::sockaddr_in6 = std::mem::zeroed(); + let mut length = size_of::() as libc::socklen_t; + if libc::getsockopt( + fd, + libc::SOL_IPV6, + 80, // IP6T_SO_ORIGINAL_DST + std::ptr::addr_of_mut!(address).cast(), + &mut length, + ) != 0 + { + return Err(std::io::Error::last_os_error()); + } + Ok(SocketAddr::new( + IpAddr::V6(std::net::Ipv6Addr::from(address.sin6_addr.s6_addr)), + u16::from_be(address.sin6_port), + )) + } +} + +#[cfg(target_os = "linux")] +fn emit_transparent_mapping_denial( + workload: SocketAddr, + original: SocketAddr, + error: MappingLookupError, +) { + let detail = match error { + MappingLookupError::Missing => "transparent_tcp_mapping_missing", + MappingLookupError::Expired => "transparent_tcp_mapping_expired", + MappingLookupError::StalePolicy => "transparent_tcp_mapping_stale_policy", + MappingLookupError::PortMismatch => "transparent_tcp_port_mismatch", + MappingLookupError::EndpointMismatch + | MappingLookupError::InvalidMapping + | MappingLookupError::LockPoisoned => "transparent_tcp_destination_denied", + }; + ocsf_emit!( + NetworkActivityBuilder::new(openshell_ocsf::ctx::ctx()) + .activity(ActivityId::Open) + .action(ActionId::Denied) + .disposition(DispositionId::Blocked) + .severity(SeverityId::Medium) + .status(StatusId::Failure) + .dst_endpoint(Endpoint::from_ip(original.ip(), original.port())) + .src_endpoint_addr(workload.ip(), workload.port()) + .message(format!("Transparent TCP denied: {error}")) + .status_detail(detail) + .build() + ); +} + +#[cfg(target_os = "linux")] +fn emit_transparent_policy_denial( + decision: &EgressDecision, + workload: SocketAddr, + host: &str, + port: u16, +) { + let status_detail = if matches!(decision.action, NetworkAction::Deny { .. }) { + "transparent_tcp_identity_denied" + } else { + "transparent_tcp_destination_denied" + }; + let binary = decision + .binary + .as_ref() + .map_or("-".to_string(), |path| path.display().to_string()); + let pid = decision + .binary_pid + .map_or("-".to_string(), |pid| pid.to_string()); + ocsf_emit!( + NetworkActivityBuilder::new(openshell_ocsf::ctx::ctx()) + .activity(ActivityId::Open) + .action(ActionId::Denied) + .disposition(DispositionId::Blocked) + .severity(SeverityId::Medium) + .status(StatusId::Failure) + .dst_endpoint(Endpoint::from_domain(host, port)) + .src_endpoint_addr(workload.ip(), workload.port()) + .actor_process(Process::from_bypass(&binary, &pid, "-")) + .firewall_rule("-", "opa") + .message(format!("Transparent TCP denied {host}:{port}")) + .status_detail(status_detail) + .build() + ); +} + fn emit_activity(tx: &Option, denied: bool, deny_group: &'static str) { if let Some(tx) = tx { let _ = try_record_activity(tx, denied, deny_group); @@ -3131,7 +3501,7 @@ fn is_cloud_metadata_ip(ip: IpAddr) -> bool { /// entry exists, the entry cannot be parsed, or the mapped IP is a cloud /// metadata address. #[cfg(any(target_os = "linux", test))] -fn detect_trusted_host_gateway() -> Option { +pub(crate) fn detect_trusted_host_gateway() -> Option { let contents = std::fs::read_to_string("/etc/hosts").ok()?; let ips = parse_hosts_file_for_host(&contents, "host.openshell.internal"); @@ -3179,7 +3549,7 @@ fn detect_trusted_host_gateway() -> Option { } #[cfg(not(any(target_os = "linux", test)))] -fn detect_trusted_host_gateway() -> Option { +pub(crate) fn detect_trusted_host_gateway() -> Option { None } diff --git a/crates/openshell-supervisor-network/src/proxy/egress.rs b/crates/openshell-supervisor-network/src/proxy/egress.rs index 2d88d74995..314596b048 100644 --- a/crates/openshell-supervisor-network/src/proxy/egress.rs +++ b/crates/openshell-supervisor-network/src/proxy/egress.rs @@ -73,8 +73,8 @@ impl EndpointDecision { pub(super) enum EgressTransport { Connect, ForwardHttp, - /// Future transparent TCP adapter fed by the policy DNS registry. - #[allow(dead_code, reason = "constructed when transparent TCP adapter lands")] + /// Transparent TCP adapter fed by the policy DNS registry. + #[cfg_attr(not(target_os = "linux"), allow(dead_code))] TransparentTcp, } @@ -101,7 +101,7 @@ impl EgressIntent { Self::new(EgressTransport::ForwardHttp, host, port) } - #[cfg(test)] + #[cfg_attr(not(target_os = "linux"), allow(dead_code))] pub(super) fn transparent_tcp(host: String, port: u16) -> Self { Self { transport: EgressTransport::TransparentTcp, diff --git a/crates/openshell-supervisor-network/src/run.rs b/crates/openshell-supervisor-network/src/run.rs index 58891ec474..a9170ceee7 100644 --- a/crates/openshell-supervisor-network/src/run.rs +++ b/crates/openshell-supervisor-network/src/run.rs @@ -38,6 +38,108 @@ use crate::opa::OpaEngine; use crate::policy_local::PolicyLocalContext; use crate::proxy::ProxyHandle; +#[cfg(target_os = "linux")] +pub struct TransparentRuntimeSetup { + pub listeners: Vec, + pub dns_udp: tokio::net::UdpSocket, + pub dns_tcp: tokio::net::TcpListener, + config: crate::policy_dns::PolicyDnsRuntimeConfig, +} + +#[cfg(target_os = "linux")] +impl TransparentRuntimeSetup { + /// Build one boot-scoped synthetic allocation epoch. The epoch advances + /// before workload execution, so addresses cached across a supervisor + /// restart fall outside the newly installed capture ranges. + /// + /// # Errors + /// + /// Returns an error when the epoch cannot be read or atomically persisted, + /// or when the derived synthetic pools are invalid. + pub fn new( + listeners: Vec, + dns_udp: tokio::net::UdpSocket, + dns_tcp: tokio::net::TcpListener, + sandbox_id: Option<&str>, + ) -> Result { + let epoch = advance_allocation_epoch( + std::path::Path::new("/run/openshell/policy-dns-epoch"), + sandbox_id, + )?; + Ok(Self { + listeners, + dns_udp, + dns_tcp, + config: crate::policy_dns::PolicyDnsRuntimeConfig::for_epoch(epoch)?, + }) + } + + #[must_use] + pub fn synthetic_cidrs(&self) -> (String, String) { + ( + self.config.ipv4_cidr.to_string(), + self.config.ipv6_cidr.to_string(), + ) + } +} + +#[cfg(target_os = "linux")] +fn advance_allocation_epoch(path: &std::path::Path, sandbox_id: Option<&str>) -> Result { + use miette::{IntoDiagnostic, WrapErr}; + use std::io::Write as _; + + let seed = sandbox_id.map_or(0, |value| { + value + .as_bytes() + .iter() + .fold(0xcbf2_9ce4_8422_2325, |hash, byte| { + (hash ^ u64::from(*byte)).wrapping_mul(0x0000_0100_0000_01b3) + }) + }); + let previous = match std::fs::read_to_string(path) { + Ok(value) => value + .trim() + .parse::() + .into_diagnostic() + .wrap_err("policy DNS allocation epoch is invalid")?, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => seed, + Err(error) => { + return Err(error) + .into_diagnostic() + .wrap_err("failed to read policy DNS allocation epoch"); + } + }; + let epoch = previous.wrapping_add(1); + let parent = path + .parent() + .ok_or_else(|| miette::miette!("policy DNS allocation epoch has no parent directory"))?; + std::fs::create_dir_all(parent) + .into_diagnostic() + .wrap_err("failed to create policy DNS runtime directory")?; + let temporary = parent.join(format!( + ".policy-dns-epoch-{}-{}", + std::process::id(), + uuid::Uuid::new_v4() + )); + let result = (|| -> std::io::Result<()> { + let mut file = std::fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(&temporary)?; + writeln!(file, "{epoch}")?; + file.sync_all()?; + std::fs::rename(&temporary, path)?; + std::fs::File::open(parent)?.sync_all() + })(); + if result.is_err() { + let _ = std::fs::remove_file(&temporary); + } + result + .into_diagnostic() + .wrap_err("failed to atomically persist policy DNS allocation epoch")?; + Ok(epoch) +} + /// Handles and values produced by [`run_networking`] that the rest of /// `run_sandbox` consumes. /// @@ -53,6 +155,10 @@ pub struct Networking { /// loop so it can publish updated `SandboxPolicy` snapshots that the /// `policy.local` route handler returns to the workload. pub policy_local_ctx: Arc, + #[cfg(target_os = "linux")] + _policy_dns: Option, + #[cfg(target_os = "linux")] + _transparent_tcp: Option, } /// Set up the networking stack: ephemeral CA + TLS state, proxy server, @@ -90,6 +196,7 @@ pub async fn run_networking( agent_proposals: AgentProposals, workspace_rx: tokio::sync::watch::Receiver, upstream_proxy_args: &crate::upstream_proxy::UpstreamProxyArgs, + #[cfg(target_os = "linux")] transparent_runtime: Option, ) -> Result { // Build the policy-local route context. The orchestrator's policy poll // loop also holds an `Arc` clone (via `Networking::policy_local_ctx`) so @@ -100,7 +207,7 @@ pub async fn run_networking( sandbox_name .map(str::to_string) .or_else(|| sandbox_id.map(str::to_string)), - agent_proposals, + agent_proposals.clone(), workspace_rx, )); @@ -110,6 +217,10 @@ pub async fn run_networking( // the race where an in-flight request observes a generation transition // during the OPA engine reload. let (engine_ready_tx, engine_ready_rx) = tokio::sync::watch::channel(false); + #[cfg(target_os = "linux")] + let transparent_engine_ready_rx = engine_ready_rx.clone(); + #[cfg(target_os = "linux")] + let policy_dns_engine_ready_rx = engine_ready_rx.clone(); // Spawn a task to resolve policy binary symlinks once the workload's mount // namespace becomes accessible via /proc//root/. The task starts @@ -312,8 +423,8 @@ pub async fn run_networking( inference_ctx, Some(provider_credentials.clone()), Some(policy_local_ctx.clone()), - denial_tx, - activity_tx, + denial_tx.clone(), + activity_tx.clone(), engine_ready_rx, upstream_proxy_args, ) @@ -323,9 +434,70 @@ pub async fn run_networking( None }; + #[cfg(target_os = "linux")] + let (policy_dns, transparent_tcp) = if let Some(runtime) = transparent_runtime { + let engine = opa_engine + .cloned() + .ok_or_else(|| miette::miette!("transparent TCP requires an OPA policy engine"))?; + let cache = identity_cache + .clone() + .ok_or_else(|| miette::miette!("transparent TCP requires a process identity cache"))?; + let trusted_gateway = crate::proxy::detect_trusted_host_gateway(); + let dns = crate::policy_dns::PolicyDnsRuntime::start( + engine.clone(), + runtime.dns_udp, + runtime.dns_tcp, + trusted_gateway, + runtime.config, + policy_dns_engine_ready_rx, + )?; + let transparent = crate::proxy::TransparentTcpHandle::start( + runtime.listeners, + dns.store.clone(), + engine, + cache, + entrypoint_pid, + agent_proposals, + denial_tx, + activity_tx, + upstream_proxy_args, + transparent_engine_ready_rx, + )?; + (Some(dns), Some(transparent)) + } else { + (None, None) + }; + Ok(Networking { proxy: proxy_handle, ca_file_paths, policy_local_ctx, + #[cfg(target_os = "linux")] + _policy_dns: policy_dns, + #[cfg(target_os = "linux")] + _transparent_tcp: transparent_tcp, }) } + +#[cfg(all(test, target_os = "linux"))] +mod transparent_runtime_tests { + use super::*; + + #[test] + fn allocation_epoch_advances_across_restart() { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("epoch"); + let first = advance_allocation_epoch(&path, Some("sandbox-a")).unwrap(); + let second = advance_allocation_epoch(&path, Some("sandbox-a")).unwrap(); + assert_eq!(second, first + 1); + } + + #[test] + fn invalid_allocation_epoch_fails_closed() { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("epoch"); + std::fs::write(&path, "corrupt\n").unwrap(); + let error = advance_allocation_epoch(&path, Some("sandbox-a")).unwrap_err(); + assert!(error.to_string().contains("allocation epoch is invalid")); + } +} diff --git a/crates/openshell-supervisor-process/Cargo.toml b/crates/openshell-supervisor-process/Cargo.toml index 7b91887588..bbd777da5b 100644 --- a/crates/openshell-supervisor-process/Cargo.toml +++ b/crates/openshell-supervisor-process/Cargo.toml @@ -18,6 +18,7 @@ openshell-policy = { path = "../openshell-policy" } anyhow = { workspace = true } base64 = { workspace = true } hex = "0.4" +ipnet = "2" miette = { workspace = true } nix = { workspace = true } rand = "0.10" @@ -39,6 +40,7 @@ rustix = { workspace = true } capctl = "0.2.4" landlock = "0.4" seccompiler = "0.5" +socket2 = { workspace = true } tempfile = "3" [dev-dependencies] diff --git a/crates/openshell-supervisor-process/src/netns/mod.rs b/crates/openshell-supervisor-process/src/netns/mod.rs index bd934da14e..91d64762bc 100644 --- a/crates/openshell-supervisor-process/src/netns/mod.rs +++ b/crates/openshell-supervisor-process/src/netns/mod.rs @@ -21,6 +21,8 @@ use uuid::Uuid; const SUBNET_PREFIX: &str = "10.200.0"; const HOST_IP_SUFFIX: u8 = 1; const SANDBOX_IP_SUFFIX: u8 = 2; +pub const POLICY_DNS_PORT: u16 = 53; +pub const TRANSPARENT_TCP_PORT: u16 = 15_001; const IP_SEARCH_PATHS: &[&str] = &["/usr/sbin/ip", "/sbin/ip", "/usr/bin/ip", "/bin/ip"]; const NSENTER_SEARCH_PATHS: &[&str] = &[ "/usr/bin/nsenter", @@ -318,6 +320,189 @@ impl NetworkNamespace { Ok(()) } + /// Replace the ordinary bypass fence with the policy-DNS and transparent + /// TCP ruleset. This is fail-closed: callers must not release workload + /// execution unless every required rule was installed. + pub fn install_transparent_tcp_rules( + &self, + proxy_port: u16, + synthetic_ipv4_cidr: &str, + synthetic_ipv6_cidr: &str, + ) -> Result<()> { + self.validate_synthetic_pool_routes(synthetic_ipv4_cidr, synthetic_ipv6_cidr)?; + // The inner namespace has an IPv4 default route, but not an IPv6 + // default route. Install only the active synthetic IPv6 epoch so the + // kernel reaches the nft OUTPUT hook; REDIRECT then reroutes it to + // the local transparent listener. + run_ip_netns( + &self.name, + &["-6", "route", "add", synthetic_ipv6_cidr, "dev", "lo"], + )?; + let nft_path = find_nft().ok_or_else(|| { + miette::miette!( + "trusted nft helper not found; policy DNS and transparent TCP require nftables" + ) + })?; + let host_ip = self.host_ip.to_string(); + let log_prefix = format!("openshell:bypass:{}:", self.name); + let commands = nft_ruleset::generate_transparent_tcp_commands( + &host_ip, + proxy_port, + POLICY_DNS_PORT, + TRANSPARENT_TCP_PORT, + synthetic_ipv4_cidr, + synthetic_ipv6_cidr, + Some(&log_prefix), + ); + run_nft_commands_netns(&self.name, &nft_path, &commands)?; + openshell_ocsf::ocsf_emit!( + openshell_ocsf::ConfigStateChangeBuilder::new(openshell_ocsf::ctx::ctx()) + .severity(openshell_ocsf::SeverityId::Informational) + .status(openshell_ocsf::StatusId::Success) + .state(openshell_ocsf::StateId::Enabled, "installed") + .message(format!( + "Policy DNS and transparent TCP capture installed [ns:{}]", + self.name + )) + .build() + ); + Ok(()) + } + + fn validate_synthetic_pool_routes( + &self, + synthetic_ipv4_cidr: &str, + synthetic_ipv6_cidr: &str, + ) -> Result<()> { + let reserved = [ + synthetic_ipv4_cidr + .parse::() + .into_diagnostic()?, + synthetic_ipv6_cidr + .parse::() + .into_diagnostic()?, + ]; + for family in ["-4", "-6"] { + let routes = + run_ip_netns_output(&self.name, &[family, "route", "show", "table", "all"])?; + if let Some((route, pool)) = first_route_overlap(&routes, &reserved) { + return Err(miette::miette!( + "synthetic address pool {pool} overlaps workload route {route}; refusing to enable policy DNS" + )); + } + } + Ok(()) + } + + /// Bind IPv4 and IPv6 transparent listeners inside the workload network + /// namespace without moving an async runtime worker into that namespace. + pub async fn bind_transparent_tcp_listeners( + &self, + ) -> std::io::Result> { + let ns_fd = self + .ns_fd + .ok_or_else(|| std::io::Error::other("no namespace fd available for bind"))?; + let (tx, rx) = tokio::sync::oneshot::channel(); + std::thread::spawn(move || { + let result = (|| -> std::io::Result> { + #[allow(unsafe_code)] + if unsafe { libc::setns(ns_fd, libc::CLONE_NEWNET) } != 0 { + return Err(std::io::Error::last_os_error()); + } + let mut listeners = Vec::with_capacity(2); + for (domain, address) in [ + ( + socket2::Domain::IPV4, + format!("0.0.0.0:{TRANSPARENT_TCP_PORT}"), + ), + ( + socket2::Domain::IPV6, + format!("[::]:{TRANSPARENT_TCP_PORT}"), + ), + ] { + let socket = socket2::Socket::new( + domain, + socket2::Type::STREAM, + Some(socket2::Protocol::TCP), + )?; + socket.set_reuse_address(true)?; + if domain == socket2::Domain::IPV6 { + socket.set_only_v6(true)?; + } + let address: std::net::SocketAddr = address.parse().map_err(|error| { + std::io::Error::other(format!("invalid listener address: {error}")) + })?; + socket.bind(&address.into())?; + socket.listen(128)?; + let listener: std::net::TcpListener = socket.into(); + listener.set_nonblocking(true)?; + listeners.push(listener); + } + Ok(listeners) + })(); + let _ = tx.send(result); + }); + rx.await + .map_err(|_| std::io::Error::other("netns bind thread panicked"))?? + .into_iter() + .map(tokio::net::TcpListener::from_std) + .collect() + } + + /// Bind UDP and TCP DNS listeners inside the workload network namespace. + /// The workload keeps its image-provided resolver configuration; nftables + /// redirects port 53 to these sockets before the bypass fence runs. + pub async fn bind_policy_dns_sockets( + &self, + ) -> std::io::Result<(tokio::net::UdpSocket, tokio::net::TcpListener)> { + let ns_fd = self + .ns_fd + .ok_or_else(|| std::io::Error::other("no namespace fd available for bind"))?; + let (tx, rx) = tokio::sync::oneshot::channel(); + std::thread::spawn(move || { + let result = (|| -> std::io::Result<(std::net::UdpSocket, std::net::TcpListener)> { + #[allow(unsafe_code)] + if unsafe { libc::setns(ns_fd, libc::CLONE_NEWNET) } != 0 { + return Err(std::io::Error::last_os_error()); + } + let address: std::net::SocketAddr = format!("0.0.0.0:{POLICY_DNS_PORT}") + .parse() + .map_err(|error| { + std::io::Error::other(format!("invalid DNS listener address: {error}")) + })?; + + let udp = socket2::Socket::new( + socket2::Domain::IPV4, + socket2::Type::DGRAM, + Some(socket2::Protocol::UDP), + )?; + udp.set_reuse_address(true)?; + udp.bind(&address.into())?; + udp.set_nonblocking(true)?; + + let tcp = socket2::Socket::new( + socket2::Domain::IPV4, + socket2::Type::STREAM, + Some(socket2::Protocol::TCP), + )?; + tcp.set_reuse_address(true)?; + tcp.bind(&address.into())?; + tcp.listen(128)?; + tcp.set_nonblocking(true)?; + + Ok((udp.into(), tcp.into())) + })(); + let _ = tx.send(result); + }); + let (udp, tcp) = rx + .await + .map_err(|_| std::io::Error::other("netns DNS bind thread panicked"))??; + Ok(( + tokio::net::UdpSocket::from_std(udp)?, + tokio::net::TcpListener::from_std(tcp)?, + )) + } + /// Bind a TCP listener inside this network namespace on a dedicated thread. /// /// Spawns a short-lived OS thread that enters the namespace via `setns`, @@ -729,6 +914,10 @@ fn run_nft_commands_current_namespace( /// The supervisor's operations (addr add, link set, route add) are all /// netlink-based and do not need sysfs access. fn run_ip_netns(netns: &str, args: &[&str]) -> Result<()> { + run_ip_netns_output(netns, args).map(|_| ()) +} + +fn run_ip_netns_output(netns: &str, args: &[&str]) -> Result { let ip_path = find_trusted_binary("ip", IP_SEARCH_PATHS)?; let nsenter_path = find_trusted_binary("nsenter", NSENTER_SEARCH_PATHS)?; let ns_path = openshell_core::container_paths::netns_path(netns); @@ -757,7 +946,29 @@ fn run_ip_netns(netns: &str, args: &[&str]) -> Result<()> { )); } - Ok(()) + Ok(String::from_utf8_lossy(&output.stdout).into_owned()) +} + +fn first_route_overlap( + routes: &str, + reserved: &[ipnet::IpNet], +) -> Option<(ipnet::IpNet, ipnet::IpNet)> { + routes.lines().find_map(|line| { + line.split_whitespace().find_map(|token| { + let route = token + .parse::() + .ok() + .or_else(|| token.parse::().ok().map(ipnet::IpNet::from))?; + reserved + .iter() + .copied() + .find(|pool| { + route.addr().is_ipv4() == pool.addr().is_ipv4() + && (route.contains(&pool.network()) || pool.contains(&route.network())) + }) + .map(|pool| (route, pool)) + }) + }) } /// Run a sequence of nft commands inside a network namespace via `nsenter --net=`. @@ -965,6 +1176,28 @@ fe800000000000000000000000000001 02 40 20 80 eth0 assert!(has_non_loopback_ipv6_interface(content)); } + #[test] + fn route_overlap_detects_reserved_pool_collision() { + let reserved = [ + "198.18.1.0/25".parse().unwrap(), + "fd23:6f70:656e:1::/120".parse().unwrap(), + ]; + let routes = "default via 10.200.0.1 dev veth\n198.18.0.0/15 dev eth1\n"; + let (route, pool) = first_route_overlap(routes, &reserved).expect("collision"); + assert_eq!(route.to_string(), "198.18.0.0/15"); + assert_eq!(pool.to_string(), "198.18.1.0/25"); + } + + #[test] + fn route_overlap_ignores_default_and_unrelated_routes() { + let reserved = [ + "198.18.1.0/25".parse().unwrap(), + "fd23:6f70:656e:1::/120".parse().unwrap(), + ]; + let routes = "default via 10.200.0.1 dev veth\n10.200.0.0/24 dev veth\n"; + assert_eq!(first_route_overlap(routes, &reserved), None); + } + #[test] #[ignore = "requires root privileges"] fn test_create_and_drop_namespace() { diff --git a/crates/openshell-supervisor-process/src/netns/nft_ruleset.rs b/crates/openshell-supervisor-process/src/netns/nft_ruleset.rs index 60263e889c..6e2fddadf6 100644 --- a/crates/openshell-supervisor-process/src/netns/nft_ruleset.rs +++ b/crates/openshell-supervisor-process/src/netns/nft_ruleset.rs @@ -203,6 +203,162 @@ pub fn generate_bypass_commands( cmds } +/// Generate the combined policy-DNS, transparent-TCP, and bypass fence. +/// +/// DNS may reach only the supervisor's trusted listener. TCP addressed to the +/// reserved synthetic pools is redirected before the terminal bypass reject; +/// all other direct TCP/UDP retains the existing fast-fail behavior. +pub fn generate_transparent_tcp_commands( + host_ip: &str, + proxy_port: u16, + dns_port: u16, + transparent_port: u16, + synthetic_ipv4_cidr: &str, + synthetic_ipv6_cidr: &str, + log_prefix: Option<&str>, +) -> Vec { + let mut cmds = vec![ + nft_cmd(true, &["add", "table", "inet", "openshell_transparent"]), + nft_cmd(true, &["flush", "table", "inet", "openshell_transparent"]), + nft_cmd( + true, + &[ + "add", + "chain", + "inet", + "openshell_transparent", + "output", + "{ type nat hook output priority dstnat; policy accept; }", + ], + ), + nft_cmd( + true, + &[ + "add", + "rule", + "inet", + "openshell_transparent", + "output", + "udp", + "dport", + &dns_port.to_string(), + "redirect", + "to", + &format!(":{dns_port}"), + ], + ), + nft_cmd( + true, + &[ + "add", + "rule", + "inet", + "openshell_transparent", + "output", + "tcp", + "dport", + &dns_port.to_string(), + "redirect", + "to", + &format!(":{dns_port}"), + ], + ), + nft_cmd( + true, + &[ + "add", + "rule", + "inet", + "openshell_transparent", + "output", + "ip", + "daddr", + synthetic_ipv4_cidr, + "tcp", + "dport", + "1-65535", + "redirect", + "to", + &format!(":{transparent_port}"), + ], + ), + nft_cmd( + true, + &[ + "add", + "rule", + "inet", + "openshell_transparent", + "output", + "ip6", + "daddr", + synthetic_ipv6_cidr, + "tcp", + "dport", + "1-65535", + "redirect", + "to", + &format!(":{transparent_port}"), + ], + ), + ]; + let mut bypass = generate_bypass_commands(host_ip, proxy_port, log_prefix); + let insertion = bypass + .iter() + .position(|command| { + command.args.iter().any(|arg| arg == "log") + || command.args.iter().any(|arg| arg == "reject") + }) + .unwrap_or(bypass.len()); + let rules = [ + nft_cmd( + true, + &[ + "add", + "rule", + "inet", + "openshell_bypass", + "output", + "udp", + "dport", + &dns_port.to_string(), + "accept", + ], + ), + nft_cmd( + true, + &[ + "add", + "rule", + "inet", + "openshell_bypass", + "output", + "tcp", + "dport", + &dns_port.to_string(), + "accept", + ], + ), + nft_cmd( + true, + &[ + "add", + "rule", + "inet", + "openshell_bypass", + "output", + "tcp", + "dport", + &transparent_port.to_string(), + "accept", + ], + ), + ]; + bypass.splice(insertion..insertion, rules); + cmds.extend(bypass); + cmds +} + /// Generate nft commands for Kubernetes sidecar enforcement. /// /// The network sidecar and the process supervisor share a pod network @@ -433,6 +589,33 @@ mod tests { assert!(ct_pos < reject_pos); } + #[test] + fn transparent_rules_precede_bypass_rejects_and_scope_dns() { + let commands = generate_transparent_tcp_commands( + "10.200.0.1", + 3128, + 53, + 15001, + "198.18.0.0/24", + "fd23:6f70:656e::/48", + None, + ); + let text = all_strs(&commands); + assert!(text.contains("udp dport 53 redirect to :53")); + assert!(text.contains("tcp dport 53 redirect to :53")); + assert!(text.contains("udp dport 53 accept")); + assert!(text.contains("ip daddr 198.18.0.0/24 tcp dport 1-65535 redirect to :15001")); + assert!( + text.contains("ip6 daddr fd23:6f70:656e::/48 tcp dport 1-65535 redirect to :15001") + ); + assert!( + text.find("tcp dport 15001 accept").unwrap() + < text + .find("meta nfproto ipv4 meta l4proto tcp reject") + .unwrap() + ); + } + #[test] fn both_ipv4_and_ipv6_reject_types_are_present() { let cmds = generate_bypass_commands("10.0.2.2", 8080, None); diff --git a/crates/openshell-supervisor-process/src/process.rs b/crates/openshell-supervisor-process/src/process.rs index 9270681f56..84f36dcac6 100644 --- a/crates/openshell-supervisor-process/src/process.rs +++ b/crates/openshell-supervisor-process/src/process.rs @@ -151,6 +151,7 @@ const SUPERVISOR_ONLY_ENV_VARS: &[&str] = &[ openshell_core::sandbox_env::TLS_CERT, openshell_core::sandbox_env::TLS_KEY, openshell_core::sandbox_env::PROVIDER_SPIFFE_WORKLOAD_API_SOCKET, + openshell_core::sandbox_env::NETWORK_RUNTIME_CAPABILITIES, ]; pub fn is_supervisor_only_env_var(key: &str) -> bool { From b47782e211550e06f956ec474fcbaec2c4f9e968 Mon Sep 17 00:00:00 2001 From: John Myers <9696606+johntmyers@users.noreply.github.com> Date: Wed, 12 Aug 2026 13:19:13 -0700 Subject: [PATCH 02/30] test(e2e): cover Docker transparent TCP egress Signed-off-by: John Myers <9696606+johntmyers@users.noreply.github.com> --- e2e/rust/tests/transparent_tcp.rs | 197 ++++++++++++++++++++++++++++++ 1 file changed, 197 insertions(+) create mode 100644 e2e/rust/tests/transparent_tcp.rs diff --git a/e2e/rust/tests/transparent_tcp.rs b/e2e/rust/tests/transparent_tcp.rs new file mode 100644 index 0000000000..ccbed211bd --- /dev/null +++ b/e2e/rust/tests/transparent_tcp.rs @@ -0,0 +1,197 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#![cfg(feature = "e2e")] + +use std::io::Write; +use std::process::Stdio; + +use openshell_e2e::harness::binary::openshell_cmd; +use openshell_e2e::harness::container::{SupportContainer, is_e2e_driver}; +use openshell_e2e::harness::sandbox::SandboxGuard; +use tempfile::NamedTempFile; + +const FIXTURE_ALIAS: &str = "transparent-tcp-fixture"; +const FIXTURE_PORT: u16 = 5432; + +fn write_policy() -> Result { + let mut file = NamedTempFile::new().map_err(|error| format!("create policy: {error}"))?; + let policy = format!( + r#"version: 1 +filesystem_policy: + include_workdir: true + read_only: [/usr, /lib, /proc, /dev/urandom, /app, /etc, /var/log] + read_write: [/sandbox, /tmp, /dev/null] +landlock: {{ compatibility: best_effort }} +process: {{ run_as_user: sandbox, run_as_group: sandbox }} +network_policies: + native_database: + name: native_database + endpoints: + - host: {FIXTURE_ALIAS} + port: {FIXTURE_PORT} + protocol: tcp + allowed_ips: ["10.0.0.0/8", "172.0.0.0/8", "192.168.0.0/16"] + binaries: + - path: "/**" +"# + ); + file.write_all(policy.as_bytes()) + .map_err(|error| format!("write policy: {error}"))?; + file.flush() + .map_err(|error| format!("flush policy: {error}"))?; + Ok(file) +} + +async fn run_cli(args: &[&str]) -> Result { + let output = openshell_cmd() + .args(args) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .output() + .await + .map_err(|error| format!("run openshell {}: {error}", args.join(" ")))?; + let combined = format!( + "{}{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + if output.status.success() { + Ok(combined) + } else { + Err(combined) + } +} + +async fn wait_for_sandbox_logs( + sandbox_name: &str, + expected: impl Fn(&str) -> bool, +) -> Result { + let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(10); + loop { + let logs = run_cli(&[ + "logs", + sandbox_name, + "-n", + "500", + "--since", + "2m", + "--source", + "sandbox", + ]) + .await?; + if expected(&logs) { + return Ok(logs); + } + if tokio::time::Instant::now() >= deadline { + return Err(format!( + "timed out waiting for transparent TCP logs:\n{logs}" + )); + } + tokio::time::sleep(std::time::Duration::from_millis(250)).await; + } +} + +#[tokio::test] +async fn docker_native_tcp_uses_policy_dns_and_fails_closed_on_wrong_port_and_real_ip() { + if !is_e2e_driver("docker") { + return; + } + + let fixture = SupportContainer::start_python( + FIXTURE_ALIAS, + &format!( + r#"import socket +s = socket.socket() +s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) +s.bind(('0.0.0.0', {FIXTURE_PORT})) +s.listen() +while True: + c, _ = s.accept() + data = c.recv(1024) + c.sendall(b'native-tcp-ok:' + data) + c.close() +"# + ), + FIXTURE_PORT, + ) + .await + .expect("start TCP fixture"); + let real_ip = fixture.ip().expect("fixture IP"); + let policy = write_policy().expect("write policy"); + let policy_path = policy.path().to_string_lossy().into_owned(); + let mut sandbox = SandboxGuard::create_keep_with_args( + &["--policy", &policy_path], + &["sh", "-c", "echo Ready; sleep infinity"], + "Ready", + ) + .await + .expect("create Docker sandbox"); + + let script = format!( + r#"import os, socket +for key in ('ALL_PROXY', 'HTTP_PROXY', 'HTTPS_PROXY', 'all_proxy', 'http_proxy', 'https_proxy'): + os.environ.pop(key, None) +try: + answers = socket.getaddrinfo({host:?}, {port}, type=socket.SOCK_STREAM) +except OSError as error: + resolver = open('/etc/resolv.conf', encoding='utf-8').read() + routes = open('/proc/net/route', encoding='utf-8').read() + raise RuntimeError(f'policy DNS lookup failed: {{error}}\nresolv.conf:\n{{resolver}}\nroutes:\n{{routes}}') from error +synthetic = sorted({{item[4][0] for item in answers}}) +assert any(ip.startswith('198.18.') or ip.startswith('198.19.') for ip in synthetic), synthetic +with socket.create_connection(({host:?}, {port}), timeout=10) as conn: + conn.sendall(b'probe') + assert conn.recv(1024) == b'native-tcp-ok:probe' + +def denied(host, port): + try: + with socket.create_connection((host, port), timeout=3) as conn: + conn.sendall(b'blocked') + return conn.recv(1024) != b'native-tcp-ok:blocked' + except OSError: + return True + +assert denied({host:?}, {wrong_port}) +assert denied({real_ip:?}, {port}) +print('transparent-tcp-e2e-ok') +"#, + host = FIXTURE_ALIAS, + port = FIXTURE_PORT, + wrong_port = FIXTURE_PORT + 1, + real_ip = real_ip, + ); + let output = match sandbox.exec(&["python3", "-c", &script]).await { + Ok(output) => output, + Err(error) => { + let logs = run_cli(&[ + "logs", + &sandbox.name, + "-n", + "500", + "--since", + "2m", + "--source", + "sandbox", + ]) + .await + .unwrap_or_else(|log_error| format!("failed to collect logs: {log_error}")); + panic!("exercise native TCP: {error}\nSandbox logs:\n{logs}"); + } + }; + assert!(output.contains("transparent-tcp-e2e-ok"), "{output}"); + + let logs = wait_for_sandbox_logs(&sandbox.name, |logs| { + logs.contains(&format!("ALLOWED {FIXTURE_ALIAS}:{FIXTURE_PORT}")) + && logs.contains("transparent_tcp_port_mismatch") + }) + .await + .expect("wait for sandbox logs"); + assert!( + logs.contains(&format!("ALLOWED {FIXTURE_ALIAS}:{FIXTURE_PORT}")), + "{logs}" + ); + assert!(logs.contains("transparent_tcp_port_mismatch"), "{logs}"); + + sandbox.cleanup().await; +} From f54b6c6969e18731ea4d13a250facf71122a3406 Mon Sep 17 00:00:00 2001 From: John Myers <9696606+johntmyers@users.noreply.github.com> Date: Wed, 12 Aug 2026 17:03:01 -0700 Subject: [PATCH 03/30] feat(network): correlate transparent TCP audit events Signed-off-by: John Myers <9696606+johntmyers@users.noreply.github.com> --- crates/openshell-ocsf/src/format/shorthand.rs | 20 +- .../src/policy_dns/mod.rs | 92 +++++++- .../openshell-supervisor-network/src/proxy.rs | 223 ++++++++++++++++-- .../src/upstream_proxy.rs | 33 ++- 4 files changed, 332 insertions(+), 36 deletions(-) diff --git a/crates/openshell-ocsf/src/format/shorthand.rs b/crates/openshell-ocsf/src/format/shorthand.rs index fa02c5cccf..77e6751b10 100644 --- a/crates/openshell-ocsf/src/format/shorthand.rs +++ b/crates/openshell-ocsf/src/format/shorthand.rs @@ -240,12 +240,20 @@ impl OcsfEvent { (false, true) => format!(" {action}"), (false, false) => format!(" {action}{arrow}"), }; - let message_ctx = - if detail.is_empty() && rule_ctx.is_empty() && reason_ctx.is_empty() { - message_tag(&e.base) - } else { - String::new() - }; + // Most network messages duplicate the structured action and + // destination shown above. Transparent TCP correlation is the + // exception: its message intentionally carries the logical, + // synthetic, and actual dial targets needed to follow the + // policy-DNS mapping in the human-readable audit log. + let show_correlation_message = + e.base.status_detail.as_deref() == Some("transparent_tcp_allowed"); + let message_ctx = if show_correlation_message + || (detail.is_empty() && rule_ctx.is_empty() && reason_ctx.is_empty()) + { + message_tag(&e.base) + } else { + String::new() + }; format!("NET:{activity} {sev}{detail}{rule_ctx}{reason_ctx}{message_ctx}") } diff --git a/crates/openshell-supervisor-network/src/policy_dns/mod.rs b/crates/openshell-supervisor-network/src/policy_dns/mod.rs index 11faf7c291..b7dd13ca9a 100644 --- a/crates/openshell-supervisor-network/src/policy_dns/mod.rs +++ b/crates/openshell-supervisor-network/src/policy_dns/mod.rs @@ -476,20 +476,49 @@ fn emit_dns_failure( } fn emit_mapping_publication(record: &ResolvedEndpointRecord) { - ocsf_emit!( - ConfigStateChangeBuilder::new(openshell_ocsf::ctx::ctx()) - .severity(SeverityId::Informational) - .status(StatusId::Success) - .state(StateId::Enabled, "published") - .unmapped("normalized_name", record.normalized_name.as_str()) - .unmapped("address_family", format!("{:?}", record.family)) - .unmapped("allowed_port_count", record.allowed_ports().len() as u64) - .unmapped("policy_generation", record.policy_generation) - .unmapped("mapping_generation", record.mapping_generation) - .unmapped("mapping_id", record.mapping_id.to_string()) - .message("Policy DNS resolved-endpoint mapping published") - .build() + ocsf_emit!(build_mapping_publication_event(record)); +} + +fn build_mapping_publication_event(record: &ResolvedEndpointRecord) -> openshell_ocsf::OcsfEvent { + let approved_real_ip_candidates = record + .contracts + .iter() + .flat_map(|contract| contract.pinned_addresses.iter().copied()) + .collect::>() + .into_iter() + .map(|address| address.to_string()) + .collect::>(); + let allowed_ports = record.allowed_ports().into_iter().collect::>(); + let mapping_id = record.mapping_id.to_string(); + let message = format!( + "Policy DNS mapped {} resolved={} synthetic={} ports={} mapping_id={mapping_id}", + record.normalized_name, + approved_real_ip_candidates.join(","), + record.synthetic_address, + allowed_ports + .iter() + .map(u16::to_string) + .collect::>() + .join(","), ); + + ConfigStateChangeBuilder::new(openshell_ocsf::ctx::ctx()) + .severity(SeverityId::Informational) + .status(StatusId::Success) + .state(StateId::Enabled, "published") + .unmapped("normalized_domain", record.normalized_name.as_str()) + .unmapped("address_family", format!("{:?}", record.family)) + .unmapped( + "approved_real_ip_candidates", + serde_json::json!(approved_real_ip_candidates), + ) + .unmapped("synthetic_ip", record.synthetic_address.to_string()) + .unmapped("allowed_ports", serde_json::json!(allowed_ports)) + .unmapped("policy_generation", record.policy_generation) + .unmapped("mapping_generation", record.mapping_generation) + .unmapped("mapping_id", mapping_id) + .message(message) + .build() } #[cfg(test)] @@ -641,6 +670,43 @@ process: { run_as_user: sandbox, run_as_group: sandbox } ); } + #[tokio::test] + async fn mapping_publication_ocsf_exposes_correlatable_resolution_chain() { + let service = service( + BASE_POLICY, + vec!["10.2.3.5".parse().unwrap(), "10.2.3.4".parse().unwrap()], + ); + let now = Instant::now(); + let answer = service + .answer_query("DB.EXAMPLE.", AddressFamily::Ipv4, now) + .await + .unwrap(); + let mapping = service + .store + .lookup(answer.address, 5432, answer.policy_generation, now) + .unwrap(); + + let event = build_mapping_publication_event(&mapping.record); + let json = event.to_json().unwrap(); + let unmapped = &json["unmapped"]; + assert_eq!(unmapped["normalized_domain"], "db.example"); + assert_eq!(unmapped["synthetic_ip"], answer.address.to_string()); + assert_eq!(unmapped["allowed_ports"], serde_json::json!([5432])); + assert_eq!( + unmapped["approved_real_ip_candidates"], + serde_json::json!(["10.2.3.4", "10.2.3.5"]) + ); + assert_eq!(unmapped["mapping_id"], answer.mapping_id.to_string()); + assert_eq!(unmapped["mapping_generation"], answer.mapping_generation); + assert_eq!(unmapped["policy_generation"], answer.policy_generation); + + let shorthand = event.format_shorthand(); + assert!(shorthand.contains("Policy DNS mapped db.example")); + assert!(shorthand.contains("resolved=10.2.3.4,10.2.3.5")); + assert!(shorthand.contains(&format!("synthetic={}", answer.address))); + assert!(shorthand.contains(&format!("mapping_id={}", answer.mapping_id))); + } + #[tokio::test] async fn wildcard_is_eligible_but_uses_public_only_destination_rules() { let yaml = BASE_POLICY.replace("db.example", "'*.example.com'"); diff --git a/crates/openshell-supervisor-network/src/proxy.rs b/crates/openshell-supervisor-network/src/proxy.rs index 5704766496..b3616e1909 100644 --- a/crates/openshell-supervisor-network/src/proxy.rs +++ b/crates/openshell-supervisor-network/src/proxy.rs @@ -667,33 +667,141 @@ async fn handle_transparent_tcp_connection( if middleware_gate == crate::l7::middleware::UninspectableTrafficGate::BypassWithFinding { crate::l7::middleware::emit_middleware_uninspectable(&ctx, "transparent tcp", false); } - let mut upstream = dial_upstream(&upstream_proxy, &host, port, connector.addrs()) + let approved_real_ip_candidates = connector.addrs().to_vec(); + let mut upstream = dial_upstream(&upstream_proxy, &host, port, &approved_real_ip_candidates) .await .into_diagnostic()?; + let upstream_socket_peer = upstream.peer_addr().into_diagnostic()?; + let (connected_real_destination, dial_mode) = match upstream.connect_target() { + Some(upstream_proxy::ConnectTarget::Ip(ip)) => ( + Some(SocketAddr::new(ip, port)), + "upstream_proxy_validated_ip", + ), + Some(upstream_proxy::ConnectTarget::Hostname) => { + (None, "upstream_proxy_hostname_resolution") + } + None => (Some(upstream_socket_peer), "direct"), + }; generation_guard.ensure_current()?; ctx.request_default_port = None; let policy_name = match &decision.action { NetworkAction::Allow { matched_policy } => matched_policy.as_deref().unwrap_or("-"), NetworkAction::Deny { .. } => "-", }; - ocsf_emit!( - NetworkActivityBuilder::new(openshell_ocsf::ctx::ctx()) - .activity(ActivityId::Open) - .action(ActionId::Allowed) - .disposition(DispositionId::Allowed) - .severity(SeverityId::Informational) - .status(StatusId::Success) - .dst_endpoint(Endpoint::from_domain(&host, port)) - .src_endpoint_addr(workload_addr.ip(), workload_addr.port()) - .firewall_rule(policy_name, "opa") - .message(format!("Transparent TCP allowed {host}:{port}")) - .status_detail("transparent_tcp_allowed") - .build() - ); + let binary = decision + .binary + .as_ref() + .map_or("-".to_string(), |path| path.display().to_string()); + let pid = decision + .binary_pid + .map_or("-".to_string(), |pid| pid.to_string()); + ocsf_emit!(build_transparent_tcp_allow_ocsf_event( + TransparentTcpAllowAudit { + workload: workload_addr, + synthetic_destination: original, + normalized_domain: &host, + approved_real_ip_candidates: &approved_real_ip_candidates, + connected_real_destination, + upstream_socket_peer, + dial_mode, + mapping_id: mapping.record.mapping_id, + mapping_generation: mapping.record.mapping_generation, + mapping_policy_generation: mapping.record.policy_generation, + authorization_policy_generation: decision.policy_generation, + binary: &binary, + pid: &pid, + policy_name, + } + )); emit_activity(&activity_tx, false, "transparent_tcp"); relay::relay_tcp(&mut client, &mut upstream, &generation_guard, &ctx).await } +#[cfg(any(target_os = "linux", test))] +struct TransparentTcpAllowAudit<'a> { + workload: SocketAddr, + synthetic_destination: SocketAddr, + normalized_domain: &'a str, + approved_real_ip_candidates: &'a [SocketAddr], + connected_real_destination: Option, + upstream_socket_peer: SocketAddr, + dial_mode: &'a str, + mapping_id: uuid::Uuid, + mapping_generation: u64, + mapping_policy_generation: u64, + authorization_policy_generation: u64, + binary: &'a str, + pid: &'a str, + policy_name: &'a str, +} + +#[cfg(any(target_os = "linux", test))] +fn build_transparent_tcp_allow_ocsf_event( + audit: TransparentTcpAllowAudit<'_>, +) -> openshell_ocsf::OcsfEvent { + let logical_destination = format!( + "{}:{}", + audit.normalized_domain, + audit.synthetic_destination.port() + ); + let mapping_id = audit.mapping_id.to_string(); + let actual_target = audit + .connected_real_destination + .map_or_else(|| "proxy-resolved".to_string(), |target| target.to_string()); + let message = format!( + "Transparent TCP mapping_id={mapping_id} synthetic={} real={actual_target}", + audit.synthetic_destination, + ); + let approved_real_ip_candidates = audit + .approved_real_ip_candidates + .iter() + .map(ToString::to_string) + .collect::>(); + let mut builder = NetworkActivityBuilder::new(openshell_ocsf::ctx::ctx()) + .activity(ActivityId::Open) + .action(ActionId::Allowed) + .disposition(DispositionId::Allowed) + .severity(SeverityId::Informational) + .status(StatusId::Success) + .dst_endpoint(Endpoint::from_domain( + audit.normalized_domain, + audit.synthetic_destination.port(), + )) + .src_endpoint_addr(audit.workload.ip(), audit.workload.port()) + .actor_process(Process::from_bypass(audit.binary, audit.pid, "")) + .firewall_rule(audit.policy_name, "opa") + .unmapped("matched_policy", audit.policy_name) + .unmapped("normalized_domain", audit.normalized_domain) + .unmapped("logical_destination", logical_destination) + .unmapped( + "synthetic_destination", + audit.synthetic_destination.to_string(), + ) + .unmapped( + "approved_real_ip_candidates", + serde_json::json!(approved_real_ip_candidates), + ) + .unmapped( + "upstream_socket_peer", + audit.upstream_socket_peer.to_string(), + ) + .unmapped("dial_mode", audit.dial_mode) + .unmapped("mapping_id", mapping_id) + .unmapped("mapping_generation", audit.mapping_generation) + .unmapped("policy_generation", audit.mapping_policy_generation) + .unmapped("mapping_policy_generation", audit.mapping_policy_generation) + .unmapped( + "authorization_policy_generation", + audit.authorization_policy_generation, + ) + .message(message) + .status_detail("transparent_tcp_allowed"); + if let Some(destination) = audit.connected_real_destination { + builder = builder.unmapped("connected_real_destination", destination.to_string()); + } + builder.build() +} + #[cfg(target_os = "linux")] fn original_destination(stream: &TcpStream) -> std::io::Result { use std::os::fd::AsRawFd; @@ -6785,6 +6893,91 @@ network_policies: ); } + #[test] + fn transparent_tcp_allow_ocsf_exposes_correlated_dns_and_dial_chain() { + let mapping_id = uuid::Uuid::new_v4(); + let event = build_transparent_tcp_allow_ocsf_event(TransparentTcpAllowAudit { + workload: "127.0.0.1:45123".parse().unwrap(), + synthetic_destination: "198.18.0.7:6379".parse().unwrap(), + normalized_domain: "redis.openshell.demo", + approved_real_ip_candidates: &[ + "172.18.0.4:6379".parse().unwrap(), + "172.18.0.5:6379".parse().unwrap(), + ], + connected_real_destination: Some("172.18.0.5:6379".parse().unwrap()), + upstream_socket_peer: "172.18.0.5:6379".parse().unwrap(), + dial_mode: "direct", + mapping_id, + mapping_generation: 4, + mapping_policy_generation: 7, + authorization_policy_generation: 7, + binary: "/sandbox/.venv/bin/python3", + pid: "42", + policy_name: "redis", + }); + let json = event.to_json().unwrap(); + + assert_eq!(json["actor"]["process"]["pid"], 42); + assert_eq!( + json["actor"]["process"]["name"], + "/sandbox/.venv/bin/python3" + ); + assert!(json["actor"]["process"].get("parent_process").is_none()); + assert_eq!(json["dst_endpoint"]["domain"], "redis.openshell.demo"); + assert_eq!(json["firewall_rule"]["name"], "redis"); + assert_eq!(json["unmapped"]["synthetic_destination"], "198.18.0.7:6379"); + assert_eq!( + json["unmapped"]["connected_real_destination"], + "172.18.0.5:6379" + ); + assert_eq!( + json["unmapped"]["approved_real_ip_candidates"], + serde_json::json!(["172.18.0.4:6379", "172.18.0.5:6379"]) + ); + assert_eq!(json["unmapped"]["mapping_id"], mapping_id.to_string()); + assert_eq!(json["unmapped"]["mapping_generation"], 4); + assert_eq!(json["unmapped"]["policy_generation"], 7); + assert_eq!(json["unmapped"]["mapping_policy_generation"], 7); + assert_eq!(json["unmapped"]["matched_policy"], "redis"); + assert_eq!(json["unmapped"]["dial_mode"], "direct"); + + let shorthand = event.format_shorthand(); + assert!(shorthand.contains("/sandbox/.venv/bin/python3(42)")); + assert!(shorthand.contains("redis.openshell.demo:6379")); + assert!(shorthand.contains("synthetic=198.18.0.7:6379")); + assert!(shorthand.contains("real=172.18.0.5:6379")); + assert!(shorthand.contains(&format!("mapping_id={mapping_id}"))); + } + + #[test] + fn transparent_tcp_proxy_hostname_audit_does_not_claim_proxy_peer_is_destination() { + let event = build_transparent_tcp_allow_ocsf_event(TransparentTcpAllowAudit { + workload: "127.0.0.1:45123".parse().unwrap(), + synthetic_destination: "198.18.0.7:6379".parse().unwrap(), + normalized_domain: "redis.openshell.demo", + approved_real_ip_candidates: &["172.18.0.4:6379".parse().unwrap()], + connected_real_destination: None, + upstream_socket_peer: "192.0.2.20:3128".parse().unwrap(), + dial_mode: "upstream_proxy_hostname_resolution", + mapping_id: uuid::Uuid::new_v4(), + mapping_generation: 4, + mapping_policy_generation: 7, + authorization_policy_generation: 7, + binary: "/usr/bin/redis-cli", + pid: "43", + policy_name: "redis", + }); + let json = event.to_json().unwrap(); + + assert!(json["unmapped"].get("connected_real_destination").is_none()); + assert_eq!(json["unmapped"]["upstream_socket_peer"], "192.0.2.20:3128"); + assert_eq!( + json["unmapped"]["dial_mode"], + "upstream_proxy_hostname_resolution" + ); + assert!(event.format_shorthand().contains("real=proxy-resolved")); + } + #[test] fn forward_ocsf_events_omit_queries_and_credential_key_names() { let peer = "127.0.0.1:45123".parse().unwrap(); diff --git a/crates/openshell-supervisor-network/src/upstream_proxy.rs b/crates/openshell-supervisor-network/src/upstream_proxy.rs index 628397bc74..e11d886375 100644 --- a/crates/openshell-supervisor-network/src/upstream_proxy.rs +++ b/crates/openshell-supervisor-network/src/upstream_proxy.rs @@ -581,6 +581,9 @@ pub struct PrefixedStream { prefix: Vec, /// Read offset into `prefix`. pos: usize, + /// CONNECT target selected for a corporate-proxy tunnel. Direct streams + /// leave this unset; their socket peer is the destination itself. + connect_target: Option, } impl PrefixedStream { @@ -591,6 +594,7 @@ impl PrefixedStream { inner, prefix, pos: 0, + connect_target: None, } } @@ -599,6 +603,25 @@ impl PrefixedStream { pub fn without_prefix(inner: TcpStream) -> Self { Self::new(inner, Vec::new()) } + + fn with_connect_target(mut self, target: ConnectTarget) -> Self { + self.connect_target = Some(target); + self + } + + /// Return the connected socket peer. For a proxied tunnel this is the + /// corporate proxy, not the ultimate destination. + pub fn peer_addr(&self) -> std::io::Result { + self.inner.peer_addr() + } + + /// Return the HTTP CONNECT target for a proxied tunnel. An IP target is + /// the exact validated destination selected by the fallback loop; a + /// hostname target means the corporate proxy performs resolution. + #[must_use] + pub fn connect_target(&self) -> Option { + self.connect_target + } } impl AsyncRead for PrefixedStream { @@ -801,7 +824,8 @@ async fn connect_via_inner( let mut stream = TcpStream::connect((endpoint.host.as_str(), endpoint.port)).await?; set_tcp_nodelay_best_effort(&stream); - let target = match target { + let connect_target = target; + let target = match connect_target { ConnectTarget::Ip(IpAddr::V6(ip)) => format!("[{ip}]:{port}"), ConnectTarget::Ip(ip) => format!("{ip}:{port}"), ConnectTarget::Hostname if host.contains(':') => format!("[{host}]:{port}"), @@ -859,7 +883,7 @@ async fn connect_via_inner( ); buf.truncate(used); let overflow = buf.split_off(header_end); - Ok(PrefixedStream::new(stream, overflow)) + Ok(PrefixedStream::new(stream, overflow).with_connect_target(connect_target)) } Some(code) => Err(IoError::other(format!( "upstream proxy {} refused CONNECT to {target}: HTTP {code}", @@ -1553,6 +1577,11 @@ mod tests { let stream = connect_via_validated(&endpoint, "api.example.com", 443, &addrs) .await .unwrap(); + assert!(matches!( + stream.connect_target(), + Some(ConnectTarget::Ip(ip)) if ip == addrs[1].ip() + )); + assert_eq!(stream.peer_addr().unwrap(), addr); let (first, second, _accepted) = handle.await.unwrap(); drop(stream); assert!( From 9fd92ab801e1f366a40fff42d3f3099840fe05e0 Mon Sep 17 00:00:00 2001 From: John Myers <9696606+johntmyers@users.noreply.github.com> Date: Wed, 12 Aug 2026 17:03:58 -0700 Subject: [PATCH 04/30] docs(examples): add transparent TCP Redis demo Signed-off-by: John Myers <9696606+johntmyers@users.noreply.github.com> --- examples/transparent-tcp-redis/README.md | 77 ++++++++++ examples/transparent-tcp-redis/demo.sh | 131 ++++++++++++++++++ examples/transparent-tcp-redis/policy.yaml | 25 ++++ .../transparent-tcp-redis/redis_client.py | 78 +++++++++++ 4 files changed, 311 insertions(+) create mode 100644 examples/transparent-tcp-redis/README.md create mode 100755 examples/transparent-tcp-redis/demo.sh create mode 100644 examples/transparent-tcp-redis/policy.yaml create mode 100644 examples/transparent-tcp-redis/redis_client.py diff --git a/examples/transparent-tcp-redis/README.md b/examples/transparent-tcp-redis/README.md new file mode 100644 index 0000000000..2c1cc7735c --- /dev/null +++ b/examples/transparent-tcp-redis/README.md @@ -0,0 +1,77 @@ +# Transparent TCP Redis + +This example connects a Docker-backed OpenShell sandbox to Redis with a native +TCP client. It does not use the HTTP forward proxy. + +The demo: + +1. Starts Redis on the OpenShell-managed Docker network. +2. Creates a sandbox with an endpoint that explicitly uses `protocol: tcp`. +3. Resolves the policy hostname to an ephemeral synthetic address. +4. Opens a native TCP socket and runs Redis `PING`, `SET`, `GET`, and `DEL` commands. +5. Prints the sandbox log stream, including OCSF DNS and TCP decisions. +6. Deletes the sandbox and Redis container, including after a failure. + +OpenShell authorizes the hostname and port before policy DNS publishes the +synthetic address. When the client connects, OpenShell maps that address back +to the approved endpoint, rechecks the process and policy, and dials a pinned +real Redis address. A direct connection to the Redis container IP remains +blocked. + +## Prerequisites + +- A Docker-backed OpenShell gateway built from the transparent TCP branch +- The `openshell` and `docker` commands +- Access to pull `redis:7-alpine` + +The Docker compute driver creates and uses the `openshell-docker` bridge by +default. The gateway process itself runs on the host; only sandbox supervisors +and the Redis service join this bridge. If the driver uses another network, set +`OPENSHELL_DOCKER_NETWORK` to that network's name. + +## Run the example + +From the repository root: + +```shell +examples/transparent-tcp-redis/demo.sh +``` + +Expected client output includes: + +```text +policy DNS: redis.openshell.demo -> 198.18.x.x +PING -> 'PONG' +SET -> 'OK' +GET -> 'hello-from-openshell' +DEL -> 1 +transparent TCP Redis demo passed +``` + +Before cleanup, the demo prints up to 500 recent sandbox log lines. Structured +security events are marked `[OCSF ]`, making the policy DNS publication and +transparent TCP allow or deny decisions visible alongside ordinary supervisor +logs. + +The synthetic address changes across supervisor allocation epochs. It is not +the Redis container's address and applications must not persist it. + +The demo currently requires the Docker compute driver. Other compute drivers +fail closed when a policy requests `protocol: tcp` until they implement the +required namespace-local DNS and TCP capture contract. + +The example policy allows any sandbox binary to use this one Redis endpoint so +the demo works across base images with different Python installation paths. In +a production policy, replace `/**` with the exact path of the client binary. + +## Configuration + +Override resource names or the image without editing the files: + +```shell +SANDBOX_NAME=tcp-redis-demo \ +REDIS_CONTAINER=my-openshell-redis \ +REDIS_IMAGE=redis:7-alpine \ +OPENSHELL_DOCKER_NETWORK=openshell-docker \ +examples/transparent-tcp-redis/demo.sh +``` diff --git a/examples/transparent-tcp-redis/demo.sh b/examples/transparent-tcp-redis/demo.sh new file mode 100755 index 0000000000..d66ed91137 --- /dev/null +++ b/examples/transparent-tcp-redis/demo.sh @@ -0,0 +1,131 @@ +#!/usr/bin/env bash + +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +POLICY_FILE="${SCRIPT_DIR}/policy.yaml" +CLIENT_FILE="${SCRIPT_DIR}/redis_client.py" + +SANDBOX_NAME="${SANDBOX_NAME:-tcp-redis-demo}" +REDIS_CONTAINER="${REDIS_CONTAINER:-openshell-transparent-tcp-redis-demo}" +DOCKER_NETWORK="${OPENSHELL_DOCKER_NETWORK:-openshell-docker}" +REDIS_IMAGE="${REDIS_IMAGE:-redis:7-alpine}" + +SANDBOX_CREATED=0 +REDIS_CREATED=0 + +cleanup() { + local status=$? + local sandbox_logs + trap - EXIT + + if [[ "$SANDBOX_CREATED" == "1" ]]; then + printf '\nSandbox logs (OCSF events are marked [OCSF ]):\n' + # Give the bounded gateway log stream a moment to receive the final + # network decision before fetching it and deleting the sandbox. + sleep 1 + if sandbox_logs="$(openshell logs "$SANDBOX_NAME" \ + --source sandbox \ + --since 10m \ + -n 500 2>&1)"; then + printf '%s\n' "$sandbox_logs" + else + printf 'Unable to retrieve sandbox logs:\n%s\n' "$sandbox_logs" >&2 + fi + fi + + printf '\nCleaning up...\n' + if [[ "$SANDBOX_CREATED" == "1" ]]; then + openshell sandbox delete "$SANDBOX_NAME" >/dev/null 2>&1 || true + fi + if [[ "$REDIS_CREATED" == "1" ]]; then + docker rm --force "$REDIS_CONTAINER" >/dev/null 2>&1 || true + fi + + exit "$status" +} +trap cleanup EXIT + +run() { + printf '\n$' + printf ' %q' "$@" + printf '\n' + "$@" +} + +for command in docker openshell; do + if ! command -v "$command" >/dev/null 2>&1; then + printf 'required command not found: %s\n' "$command" >&2 + exit 1 + fi +done + +if ! docker info >/dev/null 2>&1; then + printf 'Docker is not available. Start Docker and try again.\n' >&2 + exit 1 +fi + +if ! docker network inspect "$DOCKER_NETWORK" >/dev/null 2>&1; then + printf 'Docker network %q does not exist.\n' "$DOCKER_NETWORK" >&2 + printf 'Start a Docker-backed OpenShell gateway, or set OPENSHELL_DOCKER_NETWORK.\n' >&2 + exit 1 +fi + +if ! openshell sandbox list --limit 1 >/dev/null 2>&1; then + printf 'The configured OpenShell gateway is not reachable.\n' >&2 + printf 'Start or select a Docker-backed gateway and try again.\n' >&2 + exit 1 +fi + +if docker container inspect "$REDIS_CONTAINER" >/dev/null 2>&1; then + printf 'Redis container %q already exists; choose REDIS_CONTAINER or remove it.\n' "$REDIS_CONTAINER" >&2 + exit 1 +fi + +if openshell sandbox get "$SANDBOX_NAME" >/dev/null 2>&1; then + printf 'Sandbox %q already exists; choose SANDBOX_NAME or delete it.\n' "$SANDBOX_NAME" >&2 + exit 1 +fi + +printf 'Starting Redis on the OpenShell Docker network...\n' +REDIS_CREATED=1 +run docker run \ + --detach \ + --rm \ + --name "$REDIS_CONTAINER" \ + --network "$DOCKER_NETWORK" \ + --network-alias redis.openshell.demo \ + "$REDIS_IMAGE" \ + redis-server --save '' --appendonly no + +for _ in $(seq 1 30); do + if docker exec "$REDIS_CONTAINER" redis-cli ping 2>/dev/null | grep -qx PONG; then + break + fi + sleep 1 +done +if ! docker exec "$REDIS_CONTAINER" redis-cli ping 2>/dev/null | grep -qx PONG; then + printf 'Redis did not become ready.\n' >&2 + exit 1 +fi + +printf '\nCreating a Docker-backed sandbox with an explicit TCP endpoint policy...\n' +SANDBOX_CREATED=1 +run openshell sandbox create \ + --name "$SANDBOX_NAME" \ + --policy "$POLICY_FILE" \ + --upload "${CLIENT_FILE}:/sandbox" \ + --no-auto-providers \ + --no-tty \ + -- echo 'sandbox ready' + +printf '\nRunning native Redis commands from the sandbox...\n' +run openshell sandbox exec \ + --name "$SANDBOX_NAME" \ + --no-tty \ + -- python3 /sandbox/redis_client.py + +printf '\nTransparent TCP Redis example completed successfully.\n' diff --git a/examples/transparent-tcp-redis/policy.yaml b/examples/transparent-tcp-redis/policy.yaml new file mode 100644 index 0000000000..197e184191 --- /dev/null +++ b/examples/transparent-tcp-redis/policy.yaml @@ -0,0 +1,25 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +version: 1 + +filesystem_policy: + include_workdir: true + read_only: [/usr, /lib, /proc, /dev/urandom, /app, /etc, /var/log] + read_write: [/sandbox, /tmp, /dev/null] +landlock: + compatibility: best_effort + +network_policies: + redis: + name: redis-native-tcp + endpoints: + - host: redis.openshell.demo + port: 6379 + protocol: tcp + allowed_ips: + - 10.0.0.0/8 + - 172.16.0.0/12 + - 192.168.0.0/16 + binaries: + - { path: "/**" } diff --git a/examples/transparent-tcp-redis/redis_client.py b/examples/transparent-tcp-redis/redis_client.py new file mode 100644 index 0000000000..4cd9f00a89 --- /dev/null +++ b/examples/transparent-tcp-redis/redis_client.py @@ -0,0 +1,78 @@ +"""Minimal Redis client for the transparent TCP example.""" + +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import ipaddress +import socket + +HOST = "redis.openshell.demo" +PORT = 6379 +KEY = "openshell:transparent-tcp-demo" + + +def encode_command(*parts: str) -> bytes: + encoded = [part.encode() for part in parts] + request = [f"*{len(encoded)}\r\n".encode()] + for part in encoded: + request.extend((f"${len(part)}\r\n".encode(), part, b"\r\n")) + return b"".join(request) + + +def read_response(stream): + marker = stream.read(1) + if not marker: + raise RuntimeError("Redis closed the connection") + line = stream.readline().removesuffix(b"\r\n") + if marker == b"+": + return line.decode() + if marker == b":": + return int(line) + if marker == b"$": + length = int(line) + if length == -1: + return None + value = stream.read(length) + if stream.read(2) != b"\r\n": + raise RuntimeError("invalid Redis bulk response") + return value.decode() + if marker == b"-": + raise RuntimeError(f"Redis error: {line.decode()}") + raise RuntimeError(f"unsupported Redis response marker: {marker!r}") + + +def command(connection: socket.socket, stream, *parts: str): + connection.sendall(encode_command(*parts)) + result = read_response(stream) + print(f"{parts[0]} -> {result!r}") + return result + + +def main() -> None: + addresses = sorted( + {item[4][0] for item in socket.getaddrinfo(HOST, PORT, type=socket.SOCK_STREAM)} + ) + print(f"policy DNS: {HOST} -> {', '.join(addresses)}") + if not any( + ipaddress.ip_address(address) in ipaddress.ip_network("198.18.0.0/15") + for address in addresses + if ipaddress.ip_address(address).version == 4 + ): + raise RuntimeError("policy DNS did not return an IPv4 synthetic address") + + with ( + socket.create_connection((HOST, PORT), timeout=10) as connection, + connection.makefile("rb") as stream, + ): + assert command(connection, stream, "PING") == "PONG" + assert command(connection, stream, "SET", KEY, "hello-from-openshell") == "OK" + assert command(connection, stream, "GET", KEY) == "hello-from-openshell" + assert command(connection, stream, "DEL", KEY) == 1 + + print("transparent TCP Redis demo passed") + + +if __name__ == "__main__": + main() From 60087dc9e85db2540f7fb35c9a0b8115a2cf3dc3 Mon Sep 17 00:00:00 2001 From: John Myers <9696606+johntmyers@users.noreply.github.com> Date: Thu, 13 Aug 2026 08:29:17 -0700 Subject: [PATCH 05/30] docs(examples): demonstrate blocked TCP connections Signed-off-by: John Myers <9696606+johntmyers@users.noreply.github.com> --- examples/transparent-tcp-redis/README.md | 12 +++++++-- examples/transparent-tcp-redis/demo.sh | 10 ++++++- .../transparent-tcp-redis/redis_client.py | 26 +++++++++++++++++++ 3 files changed, 45 insertions(+), 3 deletions(-) diff --git a/examples/transparent-tcp-redis/README.md b/examples/transparent-tcp-redis/README.md index 2c1cc7735c..cab67aa2b0 100644 --- a/examples/transparent-tcp-redis/README.md +++ b/examples/transparent-tcp-redis/README.md @@ -9,8 +9,10 @@ The demo: 2. Creates a sandbox with an endpoint that explicitly uses `protocol: tcp`. 3. Resolves the policy hostname to an ephemeral synthetic address. 4. Opens a native TCP socket and runs Redis `PING`, `SET`, `GET`, and `DEL` commands. -5. Prints the sandbox log stream, including OCSF DNS and TCP decisions. -6. Deletes the sandbox and Redis container, including after a failure. +5. Confirms that policy blocks an unapproved hostname, the approved hostname + on the wrong port, and a direct connection to Redis's real IP. +6. Prints the sandbox log stream, including OCSF DNS and TCP decisions. +7. Deletes the sandbox and Redis container, including after a failure. OpenShell authorizes the hostname and port before policy DNS publishes the synthetic address. When the client connects, OpenShell maps that address back @@ -45,9 +47,15 @@ PING -> 'PONG' SET -> 'OK' GET -> 'hello-from-openshell' DEL -> 1 +BLOCKED (unapproved hostname): openshell-transparent-tcp-redis-demo:6379 -> gaierror +BLOCKED (wrong port): redis.openshell.demo:6380 -> RuntimeError +BLOCKED (direct real-IP dial): 172.x.x.x:6379 -> ConnectionRefusedError transparent TCP Redis demo passed ``` +The exact exception names vary by operating system and network timing. The +demo fails if any negative check receives a Redis response. + Before cleanup, the demo prints up to 500 recent sandbox log lines. Structured security events are marked `[OCSF ]`, making the policy DNS publication and transparent TCP allow or deny decisions visible alongside ordinary supervisor diff --git a/examples/transparent-tcp-redis/demo.sh b/examples/transparent-tcp-redis/demo.sh index d66ed91137..1a74af3e86 100755 --- a/examples/transparent-tcp-redis/demo.sh +++ b/examples/transparent-tcp-redis/demo.sh @@ -13,6 +13,7 @@ SANDBOX_NAME="${SANDBOX_NAME:-tcp-redis-demo}" REDIS_CONTAINER="${REDIS_CONTAINER:-openshell-transparent-tcp-redis-demo}" DOCKER_NETWORK="${OPENSHELL_DOCKER_NETWORK:-openshell-docker}" REDIS_IMAGE="${REDIS_IMAGE:-redis:7-alpine}" +REDIS_REAL_IP="" SANDBOX_CREATED=0 REDIS_CREATED=0 @@ -111,6 +112,13 @@ if ! docker exec "$REDIS_CONTAINER" redis-cli ping 2>/dev/null | grep -qx PONG; printf 'Redis did not become ready.\n' >&2 exit 1 fi +REDIS_REAL_IP="$(docker inspect \ + --format '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' \ + "$REDIS_CONTAINER")" +if [[ -z "$REDIS_REAL_IP" ]]; then + printf 'Could not determine the Redis container IP.\n' >&2 + exit 1 +fi printf '\nCreating a Docker-backed sandbox with an explicit TCP endpoint policy...\n' SANDBOX_CREATED=1 @@ -126,6 +134,6 @@ printf '\nRunning native Redis commands from the sandbox...\n' run openshell sandbox exec \ --name "$SANDBOX_NAME" \ --no-tty \ - -- python3 /sandbox/redis_client.py + -- python3 /sandbox/redis_client.py "$REDIS_REAL_IP" "$REDIS_CONTAINER" printf '\nTransparent TCP Redis example completed successfully.\n' diff --git a/examples/transparent-tcp-redis/redis_client.py b/examples/transparent-tcp-redis/redis_client.py index 4cd9f00a89..4de568ba90 100644 --- a/examples/transparent-tcp-redis/redis_client.py +++ b/examples/transparent-tcp-redis/redis_client.py @@ -7,6 +7,7 @@ import ipaddress import socket +import sys HOST = "redis.openshell.demo" PORT = 6379 @@ -50,7 +51,27 @@ def command(connection: socket.socket, stream, *parts: str): return result +def expect_connection_blocked(label: str, host: str, port: int) -> None: + try: + with ( + socket.create_connection((host, port), timeout=3) as connection, + connection.makefile("rb") as stream, + ): + connection.sendall(encode_command("PING")) + response = read_response(stream) + except (OSError, RuntimeError) as error: + print(f"BLOCKED ({label}): {host}:{port} -> {type(error).__name__}") + return + raise RuntimeError( + f"{label} unexpectedly reached Redis at {host}:{port}: {response!r}" + ) + + def main() -> None: + if len(sys.argv) != 3: + raise SystemExit("usage: redis_client.py REDIS_REAL_IP UNAPPROVED_HOSTNAME") + redis_real_ip, unapproved_hostname = sys.argv[1:] + addresses = sorted( {item[4][0] for item in socket.getaddrinfo(HOST, PORT, type=socket.SOCK_STREAM)} ) @@ -71,6 +92,11 @@ def main() -> None: assert command(connection, stream, "GET", KEY) == "hello-from-openshell" assert command(connection, stream, "DEL", KEY) == 1 + print("\nChecking connections that policy must block...") + expect_connection_blocked("unapproved hostname", unapproved_hostname, PORT) + expect_connection_blocked("wrong port", HOST, PORT + 1) + expect_connection_blocked("direct real-IP dial", redis_real_ip, PORT) + print("transparent TCP Redis demo passed") From 6bbbfd46f88301d4ec43ec7289d0233468045123 Mon Sep 17 00:00:00 2001 From: John Myers <9696606+johntmyers@users.noreply.github.com> Date: Thu, 13 Aug 2026 08:47:29 -0700 Subject: [PATCH 06/30] docs(examples): focus Redis demo audit output Signed-off-by: John Myers <9696606+johntmyers@users.noreply.github.com> --- examples/transparent-tcp-redis/README.md | 8 ++++---- examples/transparent-tcp-redis/demo.sh | 11 +++++++++-- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/examples/transparent-tcp-redis/README.md b/examples/transparent-tcp-redis/README.md index cab67aa2b0..dec8288944 100644 --- a/examples/transparent-tcp-redis/README.md +++ b/examples/transparent-tcp-redis/README.md @@ -56,10 +56,10 @@ transparent TCP Redis demo passed The exact exception names vary by operating system and network timing. The demo fails if any negative check receives a Redis response. -Before cleanup, the demo prints up to 500 recent sandbox log lines. Structured -security events are marked `[OCSF ]`, making the policy DNS publication and -transparent TCP allow or deny decisions visible alongside ordinary supervisor -logs. +Before cleanup, the demo fetches recent sandbox logs and prints only the OCSF +events relevant to this flow: policy DNS mappings and denials, transparent TCP +allows and wrong-port denials, and direct-bypass findings when the runtime can +observe them. The synthetic address changes across supervisor allocation epochs. It is not the Redis container's address and applications must not persist it. diff --git a/examples/transparent-tcp-redis/demo.sh b/examples/transparent-tcp-redis/demo.sh index 1a74af3e86..687513f363 100755 --- a/examples/transparent-tcp-redis/demo.sh +++ b/examples/transparent-tcp-redis/demo.sh @@ -20,11 +20,14 @@ REDIS_CREATED=0 cleanup() { local status=$? + local ocsf_pattern + local relevant_logs local sandbox_logs trap - EXIT if [[ "$SANDBOX_CREATED" == "1" ]]; then - printf '\nSandbox logs (OCSF events are marked [OCSF ]):\n' + printf '\nRelevant OCSF events:\n' + ocsf_pattern='\[OCSF \].*(Policy DNS mapped|Transparent TCP mapping_id=|policy_dns_ineligible|transparent_tcp_port_mismatch|BYPASS_DETECT)' # Give the bounded gateway log stream a moment to receive the final # network decision before fetching it and deleting the sandbox. sleep 1 @@ -32,7 +35,11 @@ cleanup() { --source sandbox \ --since 10m \ -n 500 2>&1)"; then - printf '%s\n' "$sandbox_logs" + if relevant_logs="$(printf '%s\n' "$sandbox_logs" | grep -E "$ocsf_pattern")"; then + printf '%s\n' "$relevant_logs" + else + printf 'No relevant policy DNS or transparent TCP events found.\n' + fi else printf 'Unable to retrieve sandbox logs:\n%s\n' "$sandbox_logs" >&2 fi From 8d0a341faa62167afcc9477793775f0b4ed9d45c Mon Sep 17 00:00:00 2001 From: John Myers <9696606+johntmyers@users.noreply.github.com> Date: Thu, 13 Aug 2026 11:01:22 -0700 Subject: [PATCH 07/30] fix(network): close transparent TCP policy bypasses Signed-off-by: John Myers <9696606+johntmyers@users.noreply.github.com> --- .../openshell-supervisor-network/src/proxy.rs | 136 ++++++++++++++++-- .../src/netns/nft_ruleset.rs | 42 +++++- e2e/rust/tests/transparent_tcp.rs | 23 +-- 3 files changed, 177 insertions(+), 24 deletions(-) diff --git a/crates/openshell-supervisor-network/src/proxy.rs b/crates/openshell-supervisor-network/src/proxy.rs index b3616e1909..3a831d7aed 100644 --- a/crates/openshell-supervisor-network/src/proxy.rs +++ b/crates/openshell-supervisor-network/src/proxy.rs @@ -619,6 +619,38 @@ async fn handle_transparent_tcp_connection( return Ok(()); } + // Authorization may race a policy reload. Re-pin the exact generation + // that produced the decision, then reacquire the DNS mapping against that + // generation before correlating endpoint identity or constructing a + // connector. This prevents combining an old DNS answer with a newer + // policy decision (or vice versa). + let generation_guard = + match relay::pin_policy_generation(&opa_engine, decision.policy_generation) { + Ok(guard) => guard, + Err(_) => { + emit_transparent_mapping_denial( + workload_addr, + original, + MappingLookupError::StalePolicy, + ); + emit_activity(&activity_tx, true, "transparent_tcp_mapping"); + return Ok(()); + } + }; + let mapping = match store.lookup( + original.ip(), + original.port(), + decision.policy_generation, + std::time::Instant::now(), + ) { + Ok(mapping) => mapping, + Err(error) => { + emit_transparent_mapping_denial(workload_addr, original, error); + emit_activity(&activity_tx, true, "transparent_tcp_mapping"); + return Ok(()); + } + }; + let endpoint_id = decision .endpoint .matched_endpoints @@ -650,7 +682,6 @@ async fn handle_transparent_tcp_connection( let connector = mapping.connector_for(&endpoint_id).await.map_err(|error| { miette::miette!("transparent TCP pinned destination is invalid: {error}") })?; - let generation_guard = relay::pin_policy_generation(&opa_engine, decision.policy_generation)?; let mut ctx = relay::http_context( &decision, None, @@ -668,9 +699,11 @@ async fn handle_transparent_tcp_connection( crate::l7::middleware::emit_middleware_uninspectable(&ctx, "transparent tcp", false); } let approved_real_ip_candidates = connector.addrs().to_vec(); - let mut upstream = dial_upstream(&upstream_proxy, &host, port, &approved_real_ip_candidates) - .await - .into_diagnostic()?; + generation_guard.ensure_current()?; + let mut upstream = + dial_transparent_upstream(&upstream_proxy, &host, port, &approved_real_ip_candidates) + .await + .into_diagnostic()?; let upstream_socket_peer = upstream.peer_addr().into_diagnostic()?; let (connected_real_destination, dial_mode) = match upstream.connect_target() { Some(upstream_proxy::ConnectTarget::Ip(ip)) => ( @@ -678,7 +711,7 @@ async fn handle_transparent_tcp_connection( "upstream_proxy_validated_ip", ), Some(upstream_proxy::ConnectTarget::Hostname) => { - (None, "upstream_proxy_hostname_resolution") + unreachable!("transparent TCP must bind corporate-proxy CONNECT to a validated address") } None => (Some(upstream_socket_peer), "direct"), }; @@ -3986,6 +4019,36 @@ async fn dial_upstream( )) } +/// Dial a policy-DNS-correlated transparent TCP destination. +/// +/// Unlike explicit proxy traffic, transparent TCP must never honor the +/// operator hostname-CONNECT compatibility mode: the corporate proxy must +/// receive one of the resolver-approved addresses so it cannot perform a +/// second, policy-bypassing DNS resolution. +#[cfg(target_os = "linux")] +async fn dial_transparent_upstream( + upstream_proxy: &Option, + host_lc: &str, + port: u16, + addrs: &[SocketAddr], +) -> std::io::Result { + if let Some(cfg) = upstream_proxy.as_ref() { + return match cfg.decision(host_lc, port, addrs) { + upstream_proxy::ProxyDecision::Proxy(endpoint) => { + upstream_proxy::connect_via_validated(endpoint, host_lc, port, addrs).await + } + upstream_proxy::ProxyDecision::Direct(direct_addrs) => { + Ok(upstream_proxy::PrefixedStream::without_prefix( + connect_tcp_nodelay_best_effort(&direct_addrs[..]).await?, + )) + } + }; + } + Ok(upstream_proxy::PrefixedStream::without_prefix( + connect_tcp_nodelay_best_effort(addrs).await?, + )) +} + /// Resolve a host:port using sandbox `/etc/hosts` first (when available), then /// reject if any resolved address is internal. /// @@ -6950,15 +7013,15 @@ network_policies: } #[test] - fn transparent_tcp_proxy_hostname_audit_does_not_claim_proxy_peer_is_destination() { + fn transparent_tcp_proxy_audit_reports_validated_connect_target() { let event = build_transparent_tcp_allow_ocsf_event(TransparentTcpAllowAudit { workload: "127.0.0.1:45123".parse().unwrap(), synthetic_destination: "198.18.0.7:6379".parse().unwrap(), normalized_domain: "redis.openshell.demo", approved_real_ip_candidates: &["172.18.0.4:6379".parse().unwrap()], - connected_real_destination: None, + connected_real_destination: Some("172.18.0.4:6379".parse().unwrap()), upstream_socket_peer: "192.0.2.20:3128".parse().unwrap(), - dial_mode: "upstream_proxy_hostname_resolution", + dial_mode: "upstream_proxy_validated_ip", mapping_id: uuid::Uuid::new_v4(), mapping_generation: 4, mapping_policy_generation: 7, @@ -6969,13 +7032,60 @@ network_policies: }); let json = event.to_json().unwrap(); - assert!(json["unmapped"].get("connected_real_destination").is_none()); - assert_eq!(json["unmapped"]["upstream_socket_peer"], "192.0.2.20:3128"); assert_eq!( - json["unmapped"]["dial_mode"], - "upstream_proxy_hostname_resolution" + json["unmapped"]["connected_real_destination"], + "172.18.0.4:6379" ); - assert!(event.format_shorthand().contains("real=proxy-resolved")); + assert_eq!(json["unmapped"]["upstream_socket_peer"], "192.0.2.20:3128"); + assert_eq!(json["unmapped"]["dial_mode"], "upstream_proxy_validated_ip"); + assert!(event.format_shorthand().contains("real=172.18.0.4:6379")); + } + + #[cfg(target_os = "linux")] + #[tokio::test] + async fn transparent_tcp_ignores_proxy_hostname_mode_and_connects_to_validated_ip() { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let proxy_addr = listener.local_addr().unwrap(); + let (request_tx, request_rx) = tokio::sync::oneshot::channel(); + let proxy = tokio::spawn(async move { + let (mut stream, _) = listener.accept().await.unwrap(); + let mut request = Vec::new(); + loop { + let mut byte = [0_u8; 1]; + stream.read_exact(&mut byte).await.unwrap(); + request.push(byte[0]); + if request.ends_with(b"\r\n\r\n") { + break; + } + } + request_tx.send(request).unwrap(); + stream + .write_all(b"HTTP/1.1 200 Connection Established\r\n\r\n") + .await + .unwrap(); + }); + let config = UpstreamProxyConfig::from_args(&upstream_proxy::UpstreamProxyArgs { + https_proxy: Some(format!("http://{proxy_addr}")), + proxy_connect_by_hostname: true, + ..Default::default() + }) + .unwrap() + .unwrap(); + let approved = "203.0.113.27:6379".parse().unwrap(); + + let stream = + dial_transparent_upstream(&Some(config), "redis.openshell.demo", 6379, &[approved]) + .await + .unwrap(); + let request = String::from_utf8(request_rx.await.unwrap()).unwrap(); + + assert!(request.starts_with("CONNECT 203.0.113.27:6379 HTTP/1.1\r\n")); + assert!(!request.contains("CONNECT redis.openshell.demo:6379")); + assert!(matches!( + stream.connect_target(), + Some(upstream_proxy::ConnectTarget::Ip(ip)) if ip == approved.ip() + )); + proxy.await.unwrap(); } #[test] diff --git a/crates/openshell-supervisor-process/src/netns/nft_ruleset.rs b/crates/openshell-supervisor-process/src/netns/nft_ruleset.rs index 6e2fddadf6..742b309267 100644 --- a/crates/openshell-supervisor-process/src/netns/nft_ruleset.rs +++ b/crates/openshell-supervisor-process/src/netns/nft_ruleset.rs @@ -13,6 +13,11 @@ //! `ct state` without `nf_conntrack`, `log` without `nf_log`) rolls back the //! entire transaction including table/chain creation. +/// Packet mark applied only to synthetic-destination TCP before REDIRECT. +/// The filter chain uses it to distinguish legitimate redirected listener +/// traffic from a direct dial to an arbitrary real address on the same port. +const TRANSPARENT_TCP_MARK: &str = "0x4f535450"; + /// A single nft command with metadata about whether it is required. pub struct NftCommand { /// The nft command arguments (e.g. `["add", "table", "inet", "openshell_bypass"]`). @@ -277,6 +282,10 @@ pub fn generate_transparent_tcp_commands( "tcp", "dport", "1-65535", + "meta", + "mark", + "set", + TRANSPARENT_TCP_MARK, "redirect", "to", &format!(":{transparent_port}"), @@ -296,6 +305,10 @@ pub fn generate_transparent_tcp_commands( "tcp", "dport", "1-65535", + "meta", + "mark", + "set", + TRANSPARENT_TCP_MARK, "redirect", "to", &format!(":{transparent_port}"), @@ -347,6 +360,9 @@ pub fn generate_transparent_tcp_commands( "inet", "openshell_bypass", "output", + "meta", + "mark", + TRANSPARENT_TCP_MARK, "tcp", "dport", &transparent_port.to_string(), @@ -604,12 +620,32 @@ mod tests { assert!(text.contains("udp dport 53 redirect to :53")); assert!(text.contains("tcp dport 53 redirect to :53")); assert!(text.contains("udp dport 53 accept")); - assert!(text.contains("ip daddr 198.18.0.0/24 tcp dport 1-65535 redirect to :15001")); + assert!(text.contains( + "ip daddr 198.18.0.0/24 tcp dport 1-65535 meta mark set 0x4f535450 redirect to :15001" + )); assert!( - text.contains("ip6 daddr fd23:6f70:656e::/48 tcp dport 1-65535 redirect to :15001") + text.contains("ip6 daddr fd23:6f70:656e::/48 tcp dport 1-65535 meta mark set 0x4f535450 redirect to :15001") ); + assert!(text.contains("meta mark 0x4f535450 tcp dport 15001 accept")); + assert!(!commands.iter().any(|command| { + command.args.ends_with(&[ + "tcp".to_string(), + "dport".to_string(), + "15001".to_string(), + "accept".to_string(), + ]) && !command.args.windows(3).any(|window| { + window + == [ + "meta".to_string(), + "mark".to_string(), + "0x4f535450".to_string(), + ] + }) + })); + assert!(text.contains("oifname lo accept")); assert!( - text.find("tcp dport 15001 accept").unwrap() + text.find("ip daddr 198.18.0.0/24 tcp dport 1-65535 meta mark set 0x4f535450 redirect to :15001") + .unwrap() < text .find("meta nfproto ipv4 meta l4proto tcp reject") .unwrap() diff --git a/e2e/rust/tests/transparent_tcp.rs b/e2e/rust/tests/transparent_tcp.rs index ccbed211bd..209ad354ad 100644 --- a/e2e/rust/tests/transparent_tcp.rs +++ b/e2e/rust/tests/transparent_tcp.rs @@ -13,6 +13,7 @@ use tempfile::NamedTempFile; const FIXTURE_ALIAS: &str = "transparent-tcp-fixture"; const FIXTURE_PORT: u16 = 5432; +const TRANSPARENT_LISTENER_PORT: u16 = 15001; fn write_policy() -> Result { let mut file = NamedTempFile::new().map_err(|error| format!("create policy: {error}"))?; @@ -101,16 +102,20 @@ async fn docker_native_tcp_uses_policy_dns_and_fails_closed_on_wrong_port_and_re let fixture = SupportContainer::start_python( FIXTURE_ALIAS, &format!( - r#"import socket -s = socket.socket() -s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) -s.bind(('0.0.0.0', {FIXTURE_PORT})) -s.listen() -while True: + r#"import socket, threading +def serve(port): + s = socket.socket() + s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + s.bind(('0.0.0.0', port)) + s.listen() + while True: c, _ = s.accept() data = c.recv(1024) c.sendall(b'native-tcp-ok:' + data) c.close() + +threading.Thread(target=serve, args=({TRANSPARENT_LISTENER_PORT},), daemon=True).start() +serve({FIXTURE_PORT}) "# ), FIXTURE_PORT, @@ -154,12 +159,14 @@ def denied(host, port): assert denied({host:?}, {wrong_port}) assert denied({real_ip:?}, {port}) +assert denied({real_ip:?}, {transparent_port}) print('transparent-tcp-e2e-ok') "#, host = FIXTURE_ALIAS, port = FIXTURE_PORT, wrong_port = FIXTURE_PORT + 1, real_ip = real_ip, + transparent_port = TRANSPARENT_LISTENER_PORT, ); let output = match sandbox.exec(&["python3", "-c", &script]).await { Ok(output) => output, @@ -182,13 +189,13 @@ print('transparent-tcp-e2e-ok') assert!(output.contains("transparent-tcp-e2e-ok"), "{output}"); let logs = wait_for_sandbox_logs(&sandbox.name, |logs| { - logs.contains(&format!("ALLOWED {FIXTURE_ALIAS}:{FIXTURE_PORT}")) + logs.contains(&format!("-> {FIXTURE_ALIAS}:{FIXTURE_PORT}")) && logs.contains("transparent_tcp_port_mismatch") }) .await .expect("wait for sandbox logs"); assert!( - logs.contains(&format!("ALLOWED {FIXTURE_ALIAS}:{FIXTURE_PORT}")), + logs.contains(&format!("-> {FIXTURE_ALIAS}:{FIXTURE_PORT}")), "{logs}" ); assert!(logs.contains("transparent_tcp_port_mismatch"), "{logs}"); From a627da9542ef125cb5f8197e699255bbc2cc475a Mon Sep 17 00:00:00 2001 From: John Myers <9696606+johntmyers@users.noreply.github.com> Date: Thu, 13 Aug 2026 11:01:50 -0700 Subject: [PATCH 08/30] fix(sandbox): reject unsupported TCP policy reloads Signed-off-by: John Myers <9696606+johntmyers@users.noreply.github.com> --- architecture/sandbox.md | 7 +- crates/openshell-sandbox/src/lib.rs | 235 +++++++++++++++++++++++ examples/transparent-tcp-redis/README.md | 5 + 3 files changed, 246 insertions(+), 1 deletion(-) diff --git a/architecture/sandbox.md b/architecture/sandbox.md index 333410623b..4059fbd1e0 100644 --- a/architecture/sandbox.md +++ b/architecture/sandbox.md @@ -93,7 +93,12 @@ instead of inheriting a new mapping. Policy reload, expiry, wrong ports, direct mappings, or pool exhaustion fail closed. Resolver injection, DNS listeners, capture rules, and the transparent listener are all ready before workload execution. A runtime that cannot provide the complete contract rejects a policy -containing explicit TCP endpoints rather than partially activating it. +containing explicit TCP endpoints rather than partially activating it. Because +that substrate is startup infrastructure, a sandbox created without explicit +TCP endpoints rejects a hot reload that introduces one and keeps its complete +previous policy active; recreating the sandbox installs the substrate before +the workload starts. A sandbox that started with the substrate may continue to +remove and re-add TCP endpoints through ordinary atomic policy reloads. Provider credential placeholders are resolved through the live provider state for each HTTP request, after destination and L7 policy admission. A static diff --git a/crates/openshell-sandbox/src/lib.rs b/crates/openshell-sandbox/src/lib.rs index 43457a5bbb..1e2ca12d3d 100644 --- a/crates/openshell-sandbox/src/lib.rs +++ b/crates/openshell-sandbox/src/lib.rs @@ -397,6 +397,8 @@ pub async fn run_sandbox( runtime_capabilities.as_deref(), openshell_core::sandbox_env::POLICY_DNS_TRANSPARENT_TCP_CAPABILITY, ); + #[cfg(not(target_os = "linux"))] + let transparent_tcp_capable = false; #[cfg(target_os = "linux")] let transparent_runtime = if transparent_tcp_requested { if !transparent_tcp_capable { @@ -458,6 +460,10 @@ pub async fn run_sandbox( } else { None }; + #[cfg(target_os = "linux")] + let transparent_tcp_substrate_ready = transparent_runtime.is_some(); + #[cfg(not(target_os = "linux"))] + let transparent_tcp_substrate_ready = false; // The denial channel is owned by the orchestrator: the proxy (in the // networking leaf) and the bypass monitor (in the process leaf) both // produce DenialEvents that the denial aggregator (orchestrator-side) @@ -734,6 +740,10 @@ pub async fn run_sandbox( extension_credentials: extension_credentials.clone(), extension_authentication_enabled: initial_extension_authentication_enabled, middleware_connector: default_middleware_connector(), + transparent_tcp: TransparentTcpReloadState { + capable: transparent_tcp_capable, + substrate_ready: transparent_tcp_substrate_ready, + }, }; tokio::spawn(async move { @@ -2403,12 +2413,14 @@ enum MiddlewareRegistryStatus { #[derive(Debug)] enum GatewayRuntimeReloadError { PolicyValidation(miette::Report), + TransparentTcpPrerequisite(miette::Report), MiddlewareRegistry(miette::Report), } #[derive(Clone, Copy, Debug, PartialEq, Eq)] enum GatewayRuntimeFailureClass { PolicyValidation, + TransparentTcpPrerequisite, MiddlewareRegistry, } @@ -2416,6 +2428,9 @@ impl GatewayRuntimeReloadError { fn class(&self) -> GatewayRuntimeFailureClass { match self { Self::PolicyValidation(_) => GatewayRuntimeFailureClass::PolicyValidation, + Self::TransparentTcpPrerequisite(_) => { + GatewayRuntimeFailureClass::TransparentTcpPrerequisite + } Self::MiddlewareRegistry(_) => GatewayRuntimeFailureClass::MiddlewareRegistry, } } @@ -2446,7 +2461,26 @@ async fn reload_gateway_policy_runtime( middleware_authentication: &MiddlewareAuthentication, middleware_registry_changed: bool, middleware_connector: &MiddlewareConnector, + transparent_tcp: TransparentTcpReloadState, ) -> std::result::Result<(), GatewayRuntimeReloadError> { + if let Some(policy) = policy + && policy_contains_explicit_tcp(policy) + { + if !transparent_tcp.capable { + return Err(GatewayRuntimeReloadError::TransparentTcpPrerequisite( + miette::miette!( + "candidate policy introduces protocol: tcp, but the runtime does not advertise transparent TCP support; previous policy remains active" + ), + )); + } + if !transparent_tcp.substrate_ready { + return Err(GatewayRuntimeReloadError::TransparentTcpPrerequisite( + miette::miette!( + "candidate policy introduces protocol: tcp, but this sandbox started without the transparent TCP substrate; recreate the sandbox to enable TCP; previous policy remains active" + ), + )); + } + } match policy { Some(policy) if middleware_registry_changed => { let registry = @@ -2469,6 +2503,20 @@ async fn reload_gateway_policy_runtime( } } +fn policy_contains_explicit_tcp(policy: &openshell_core::proto::SandboxPolicy) -> bool { + policy.network_policies.values().any(|rule| { + rule.endpoints + .iter() + .any(|endpoint| endpoint.protocol.eq_ignore_ascii_case("tcp")) + }) +} + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +struct TransparentTcpReloadState { + capable: bool, + substrate_ready: bool, +} + /// True when the installed middleware registry no longer matches the desired /// service set and must be rebuilt (reconnecting every delivered service). /// @@ -2985,6 +3033,8 @@ struct PolicyPollLoopContext { extension_credentials: openshell_extension_core::ExtensionCredentialStore, extension_authentication_enabled: bool, middleware_connector: MiddlewareConnector, + /// Immutable driver capability and startup substrate state. + transparent_tcp: TransparentTcpReloadState, } type MiddlewareConnector = Arc< @@ -3171,6 +3221,10 @@ enum GatewayRuntimeFailureDisposition { MiddlewareUnavailable { error: String, }, + TransparentTcpExpansionRejected { + error: String, + active_generation: u64, + }, } fn apply_gateway_runtime_reload_failure( @@ -3192,6 +3246,12 @@ fn apply_gateway_runtime_reload_failure( )?; Ok(GatewayRuntimeFailureDisposition::PolicyRejected { error, disposition }) } + GatewayRuntimeReloadError::TransparentTcpPrerequisite(error) => Ok( + GatewayRuntimeFailureDisposition::TransparentTcpExpansionRejected { + error: error.to_string(), + active_generation: engine.current_generation(), + }, + ), GatewayRuntimeReloadError::MiddlewareRegistry(error) => { Ok(GatewayRuntimeFailureDisposition::MiddlewareUnavailable { error: error.to_string(), @@ -3200,6 +3260,30 @@ fn apply_gateway_runtime_reload_failure( } } +fn emit_transparent_tcp_expansion_rejection( + version: u32, + policy_hash: &str, + active_generation: u64, + error: &str, +) { + let message = format!( + "Transparent TCP policy expansion rejected; previous policy IS active [version:{version} active_generation:{active_generation} error:{error}]" + ); + ocsf_emit!( + ConfigStateChangeBuilder::new(ocsf_ctx()) + .severity(SeverityId::High) + .status(StatusId::Failure) + .state(StateId::Enabled, "retained_previous_policy") + .unmapped("candidate_version", serde_json::json!(version)) + .unmapped("candidate_policy_hash", serde_json::json!(policy_hash)) + .unmapped("previous_policy_active", serde_json::json!(true)) + .unmapped("active_generation", serde_json::json!(active_generation)) + .unmapped("validation_error", serde_json::json!(error)) + .message(message) + .build() + ); +} + fn apply_policy_validation_failure( engine: &OpaEngine, configured_mode: PolicyValidationFailureMode, @@ -3686,6 +3770,7 @@ async fn run_policy_poll_loop_with_client( }, middleware_registry_changed, &ctx.middleware_connector, + ctx.transparent_tcp, ) .await; @@ -3849,6 +3934,26 @@ async fn run_policy_poll_loop_with_client( )) .build()); } + GatewayRuntimeFailureDisposition::TransparentTcpExpansionRejected { + error, + active_generation, + } => { + emit_transparent_tcp_expansion_rejection( + result.version, + &result.policy_hash, + active_generation, + &error, + ); + if policy_changed + && result.version > 0 + && result.policy_source == PolicySource::Sandbox + { + enqueue_policy_status( + &status_sender, + PolicyStatusUpdate::failed(result.version, error), + ); + } + } } } last_failed_runtime_revision = Some(failed_revision); @@ -4398,6 +4503,24 @@ filesystem_policy: openshell_policy::restrictive_default_policy() } + fn proto_tcp_policy_fixture() -> openshell_core::proto::SandboxPolicy { + openshell_policy::parse_sandbox_policy( + r#" +version: 1 +network_policies: + redis: + name: redis + endpoints: + - host: redis.example.com + port: 6379 + protocol: tcp + binaries: + - path: /usr/bin/redis-cli +"#, + ) + .expect("parse TCP policy") + } + fn settings_poll_result( policy: Option, version: u32, @@ -4548,6 +4671,7 @@ filesystem_policy: extension_credentials: openshell_extension_core::ExtensionCredentialStore::new(), extension_authentication_enabled: false, middleware_connector, + transparent_tcp: TransparentTcpReloadState::default(), } } @@ -4615,6 +4739,54 @@ filesystem_policy: handle.abort(); } + #[tokio::test] + async fn poll_rejects_first_tcp_expansion_and_reports_previous_policy_active() { + let v1 = settings_poll_result( + Some(proto_policy_fixture()), + 1, + openshell_core::proto::PolicySource::Sandbox, + ); + let v2 = settings_poll_result( + Some(proto_tcp_policy_fixture()), + 2, + openshell_core::proto::PolicySource::Sandbox, + ); + let engine = + Arc::new(OpaEngine::from_proto(&proto_policy_fixture()).expect("build OPA engine")); + let active_generation = engine.current_generation(); + let loaded_revision = LoadedPolicyRevision::from_snapshot(&v1); + let mut ctx = policy_poll_test_context( + engine.clone(), + LoadedPolicyOrigin::Gateway { + revision: Some(loaded_revision), + has_last_valid_policy: true, + }, + default_middleware_connector(), + ); + ctx.transparent_tcp = TransparentTcpReloadState { + capable: true, + substrate_ready: false, + }; + let (client, polls, mut reports) = scripted_policy_gateway(); + polls.send(v1).unwrap(); + + let handle = tokio::spawn(run_policy_poll_loop_with_client(ctx, client)); + expect_policy_report(&mut reports, 1).await; + polls.send(v2).unwrap(); + let report = timeout(Duration::from_secs(1), reports.recv()) + .await + .expect("TCP rejection report timed out") + .expect("policy reporter stopped"); + + assert_eq!(report.0, 2); + assert!(!report.1); + assert!(report.2.contains("recreate the sandbox"), "{}", report.2); + assert!(report.2.contains("previous policy remains active")); + assert_eq!(engine.current_generation(), active_generation); + assert!(engine.fail_closed_reason().is_none()); + handle.abort(); + } + #[tokio::test] async fn same_hash_ack_waits_for_failed_middleware_reconciliation_and_retries_once() { let mut v1 = settings_poll_result( @@ -5006,6 +5178,7 @@ filesystem_policy: &MiddlewareAuthentication::default(), true, &default_middleware_connector(), + TransparentTcpReloadState::default(), ) .await .expect_err("unavailable middleware must fail candidate preparation"); @@ -5026,6 +5199,68 @@ filesystem_policy: assert!(engine.fail_closed_reason().is_none()); } + #[tokio::test] + async fn tcp_policy_reload_without_startup_substrate_is_rejected_and_keeps_previous_policy() { + let engine = OpaEngine::from_proto(&proto_policy_fixture()).expect("build OPA engine"); + let active_generation = engine.current_generation(); + + let failure = reload_gateway_policy_runtime( + &engine, + Some(&proto_tcp_policy_fixture()), + 0, + &[], + false, + &default_middleware_connector(), + TransparentTcpReloadState { + capable: true, + substrate_ready: false, + }, + ) + .await + .expect_err("TCP expansion must require startup substrate"); + let disposition = apply_gateway_runtime_reload_failure( + &engine, + failure, + PolicyValidationFailureMode::FailClosed, + true, + 2, + ) + .expect("runtime prerequisite failure handling must succeed"); + + assert!(matches!( + disposition, + GatewayRuntimeFailureDisposition::TransparentTcpExpansionRejected { + active_generation: generation, + .. + } if generation == active_generation + )); + assert_eq!(engine.current_generation(), active_generation); + assert!(engine.fail_closed_reason().is_none()); + } + + #[tokio::test] + async fn tcp_policy_reload_on_unsupported_runtime_is_rejected() { + let engine = OpaEngine::from_proto(&proto_policy_fixture()).expect("build OPA engine"); + + let failure = reload_gateway_policy_runtime( + &engine, + Some(&proto_tcp_policy_fixture()), + 0, + &[], + false, + &default_middleware_connector(), + TransparentTcpReloadState::default(), + ) + .await + .expect_err("unsupported runtime must reject TCP expansion"); + + assert!(matches!( + failure, + GatewayRuntimeReloadError::TransparentTcpPrerequisite(_) + )); + assert_eq!(engine.current_generation(), 0); + } + #[test] fn policy_rejection_after_middleware_outage_is_not_deduplicated() { let engine = OpaEngine::from_strings( diff --git a/examples/transparent-tcp-redis/README.md b/examples/transparent-tcp-redis/README.md index dec8288944..4e04525da7 100644 --- a/examples/transparent-tcp-redis/README.md +++ b/examples/transparent-tcp-redis/README.md @@ -68,6 +68,11 @@ The demo currently requires the Docker compute driver. Other compute drivers fail closed when a policy requests `protocol: tcp` until they implement the required namespace-local DNS and TCP capture contract. +Create the sandbox with at least one explicit TCP endpoint. Adding the first +`protocol: tcp` endpoint to a running sandbox that started without one is +rejected atomically, with the previous policy left active; recreate the sandbox +to install the DNS and transparent TCP substrate before the workload starts. + The example policy allows any sandbox binary to use this one Redis endpoint so the demo works across base images with different Python installation paths. In a production policy, replace `/**` with the exact path of the client binary. From 531ef62e25c1234fa499b2cc6b2e644e34e1a6c3 Mon Sep 17 00:00:00 2001 From: John Myers <9696606+johntmyers@users.noreply.github.com> Date: Thu, 13 Aug 2026 11:08:24 -0700 Subject: [PATCH 09/30] fix(ci): satisfy Linux transparent TCP lints Signed-off-by: John Myers <9696606+johntmyers@users.noreply.github.com> --- .../openshell-supervisor-network/src/proxy.rs | 38 +++++++++---------- .../src/netns/mod.rs | 6 ++- 2 files changed, 21 insertions(+), 23 deletions(-) diff --git a/crates/openshell-supervisor-network/src/proxy.rs b/crates/openshell-supervisor-network/src/proxy.rs index 3a831d7aed..e7552a1b9a 100644 --- a/crates/openshell-supervisor-network/src/proxy.rs +++ b/crates/openshell-supervisor-network/src/proxy.rs @@ -624,19 +624,13 @@ async fn handle_transparent_tcp_connection( // generation before correlating endpoint identity or constructing a // connector. This prevents combining an old DNS answer with a newer // policy decision (or vice versa). - let generation_guard = - match relay::pin_policy_generation(&opa_engine, decision.policy_generation) { - Ok(guard) => guard, - Err(_) => { - emit_transparent_mapping_denial( - workload_addr, - original, - MappingLookupError::StalePolicy, - ); - emit_activity(&activity_tx, true, "transparent_tcp_mapping"); - return Ok(()); - } - }; + let Ok(generation_guard) = + relay::pin_policy_generation(&opa_engine, decision.policy_generation) + else { + emit_transparent_mapping_denial(workload_addr, original, MappingLookupError::StalePolicy); + emit_activity(&activity_tx, true, "transparent_tcp_mapping"); + return Ok(()); + }; let mapping = match store.lookup( original.ip(), original.port(), @@ -724,10 +718,10 @@ async fn handle_transparent_tcp_connection( let binary = decision .binary .as_ref() - .map_or("-".to_string(), |path| path.display().to_string()); + .map_or_else(|| "-".to_string(), |path| path.display().to_string()); let pid = decision .binary_pid - .map_or("-".to_string(), |pid| pid.to_string()); + .map_or_else(|| "-".to_string(), |pid| pid.to_string()); ocsf_emit!(build_transparent_tcp_allow_ocsf_event( TransparentTcpAllowAudit { workload: workload_addr, @@ -843,13 +837,14 @@ fn original_destination(stream: &TcpStream) -> std::io::Result { #[allow(unsafe_code)] unsafe { let mut address: libc::sockaddr_in = std::mem::zeroed(); - let mut length = size_of::() as libc::socklen_t; + let mut length = libc::socklen_t::try_from(size_of::()) + .expect("sockaddr_in size fits socklen_t"); if libc::getsockopt( fd, libc::SOL_IP, 80, // SO_ORIGINAL_DST std::ptr::addr_of_mut!(address).cast(), - &mut length, + std::ptr::addr_of_mut!(length), ) != 0 { return Err(std::io::Error::last_os_error()); @@ -865,13 +860,14 @@ fn original_destination(stream: &TcpStream) -> std::io::Result { #[allow(unsafe_code)] unsafe { let mut address: libc::sockaddr_in6 = std::mem::zeroed(); - let mut length = size_of::() as libc::socklen_t; + let mut length = libc::socklen_t::try_from(size_of::()) + .expect("sockaddr_in6 size fits socklen_t"); if libc::getsockopt( fd, libc::SOL_IPV6, 80, // IP6T_SO_ORIGINAL_DST std::ptr::addr_of_mut!(address).cast(), - &mut length, + std::ptr::addr_of_mut!(length), ) != 0 { return Err(std::io::Error::last_os_error()); @@ -928,10 +924,10 @@ fn emit_transparent_policy_denial( let binary = decision .binary .as_ref() - .map_or("-".to_string(), |path| path.display().to_string()); + .map_or_else(|| "-".to_string(), |path| path.display().to_string()); let pid = decision .binary_pid - .map_or("-".to_string(), |pid| pid.to_string()); + .map_or_else(|| "-".to_string(), |pid| pid.to_string()); ocsf_emit!( NetworkActivityBuilder::new(openshell_ocsf::ctx::ctx()) .activity(ActivityId::Open) diff --git a/crates/openshell-supervisor-process/src/netns/mod.rs b/crates/openshell-supervisor-process/src/netns/mod.rs index 91d64762bc..30411cb4df 100644 --- a/crates/openshell-supervisor-process/src/netns/mod.rs +++ b/crates/openshell-supervisor-process/src/netns/mod.rs @@ -963,8 +963,10 @@ fn first_route_overlap( .iter() .copied() .find(|pool| { - route.addr().is_ipv4() == pool.addr().is_ipv4() - && (route.contains(&pool.network()) || pool.contains(&route.network())) + let same_family = route.addr().is_ipv4() == pool.addr().is_ipv4(); + let overlaps = + route.contains(&pool.network()) || pool.contains(&route.network()); + same_family && overlaps }) .map(|pool| (route, pool)) }) From 05de9015e0a755c5f4e938ae8bba13d7d0c684e8 Mon Sep 17 00:00:00 2001 From: John Myers <9696606+johntmyers@users.noreply.github.com> Date: Thu, 13 Aug 2026 13:29:42 -0700 Subject: [PATCH 10/30] feat(podman): enable transparent TCP egress Signed-off-by: John Myers <9696606+johntmyers@users.noreply.github.com> --- .agents/skills/debug-openshell-cluster/SKILL.md | 6 +++++- .agents/skills/generate-sandbox-policy/SKILL.md | 4 ++-- .agents/skills/openshell-cli/SKILL.md | 2 +- architecture/compute-runtimes.md | 4 ++-- crates/openshell-driver-podman/NETWORKING.md | 8 ++++++++ crates/openshell-driver-podman/README.md | 4 ++++ crates/openshell-driver-podman/src/container.rs | 12 ++++++------ e2e/rust/tests/transparent_tcp.rs | 6 +++--- 8 files changed, 31 insertions(+), 15 deletions(-) diff --git a/.agents/skills/debug-openshell-cluster/SKILL.md b/.agents/skills/debug-openshell-cluster/SKILL.md index f06d3df470..a731600943 100644 --- a/.agents/skills/debug-openshell-cluster/SKILL.md +++ b/.agents/skills/debug-openshell-cluster/SKILL.md @@ -186,7 +186,7 @@ Common findings: callbacks. On an older release, set `bind_address = "127.0.0.1:17670"` or upgrade. - Supervisor image exits before printing `openshell-sandbox --version`: the image should be the scratch supervisor image from `deploy/docker/Dockerfile.supervisor` and must contain a static executable at `/openshell-sandbox`. -- A sandbox with explicit `protocol: tcp` endpoints fails before workload readiness: confirm the Docker driver supplied the `policy-dns-transparent-tcp` runtime capability and inspect supervisor logs for missing `nft`, synthetic-route overlap, or namespace-local DNS/TCP listener bind failures. Podman, Kubernetes, VM, sidecar, and out-of-tree drivers must reject this policy until they provide the complete substrate; use omitted protocol with an explicit proxy on those runtimes. +- A sandbox with explicit `protocol: tcp` endpoints fails before workload readiness: confirm the Docker or Podman driver supplied the `policy-dns-transparent-tcp` runtime capability and inspect supervisor logs for missing `nft`, synthetic-route overlap, or namespace-local DNS/TCP listener bind failures. Kubernetes, VM, sidecar, and out-of-tree drivers must reject this policy until they provide the complete substrate; use omitted protocol with an explicit proxy on those runtimes. - `mise run e2e:docker:gpu` fails with `docker info --format json did not report any discovered NVIDIA CDI GPU devices`: Docker may report `CDISpecDirs` while still having no generated NVIDIA CDI specs. Verify `.DiscoveredDevices` contains entries such as `nvidia.com/gpu=all`, verify `/etc/cdi` or `/var/run/cdi` contains a generated NVIDIA spec, and check that `nvidia-cdi-refresh.service` and `nvidia-cdi-refresh.path` from NVIDIA Container Toolkit are enabled and healthy. The service is a one-shot unit, so `inactive (dead)` can be normal after a successful run; use `systemctl status` and `journalctl` to distinguish success from a skipped or failed refresh. NVIDIA recommends enabling the path and service units, and restarting `nvidia-cdi-refresh.service` to regenerate missing or stale CDI specs. If specs are generated but Docker still reports no discovered devices, restart Docker or reload the daemon and re-check `docker info`. For source checkout development, restart the local gateway with: @@ -211,6 +211,10 @@ Common findings: - Sandbox image missing or pull denied: verify image reference and registry credentials. - Sandbox fails before readiness with an identity-resolution error: inspect the image's OCI `USER` and matching `/etc/passwd` and `/etc/group` entries, or explicitly set both process identity fields in policy. Numeric workload identities `1` through `4294967294` are accepted; root, the invalid identity sentinel, and missing identities are rejected. - Supervisor cannot call back: check callback endpoint and gateway logs. +- A sandbox with explicit `protocol: tcp` endpoints fails before readiness: + inspect supervisor logs for policy DNS port-53 binding, synthetic-route, or + nftables redirect failures. Rootless Podman must provide these primitives + inside the supervisor-owned nested network namespace; setup fails closed. - Gateway exits before becoming healthy with a callback-listener discovery error: inspect `podman info --debug`, the configured Podman network, and the host's IPv4 default route. Rootless pasta uses the private source address diff --git a/.agents/skills/generate-sandbox-policy/SKILL.md b/.agents/skills/generate-sandbox-policy/SKILL.md index 734e188783..f455590eca 100644 --- a/.agents/skills/generate-sandbox-policy/SKILL.md +++ b/.agents/skills/generate-sandbox-policy/SKILL.md @@ -46,7 +46,7 @@ For this tier, default to: inspection". Omit `protocol` for explicit-proxy clients. Use `protocol: tcp` only when the workload must use native DNS and direct socket calls, the endpoint has a valid DNS hostname, and the selected runtime - advertises policy DNS and transparent TCP support. + support (currently Docker and Podman). ### Moderate Tier (host + partial path knowledge) @@ -416,7 +416,7 @@ Evaluate the generated policy for overly broad access and **include warnings in | Condition | Warning to show | |-----------|----------------| -| **L4-only** (no `protocol`, or `protocol: tcp`) | "This policy allows all application methods and paths without inspection. An omitted protocol uses explicit-proxy behavior; `protocol: tcp` enables policy DNS and transparent TCP only on a runtime that advertises the complete substrate (currently Docker). Consider `protocol: rest` with a preset if you want HTTP method-level control." | +| **L4-only** (no `protocol`, or `protocol: tcp`) | "This policy allows all application methods and paths without inspection. An omitted protocol uses explicit-proxy behavior; `protocol: tcp` enables policy DNS and transparent TCP only on a runtime that advertises the complete substrate (currently Docker and Podman). Consider `protocol: rest` with a preset if you want HTTP method-level control." | | **`access: full`** | "This policy allows all HTTP methods (including DELETE) on all paths. If you don't need DELETE, `read-write` is safer. If you only need to read, `read-only` is the most restrictive option." | | **`access: full` + `enforcement: audit`** | "Full access in audit mode provides no actual restriction — all traffic flows through. This is effectively a monitoring-only policy." | | **`access: read-write`** when user hasn't confirmed write need | "This policy allows POST, PUT, and PATCH on all paths. If you only need to read data, `read-only` is more restrictive." | diff --git a/.agents/skills/openshell-cli/SKILL.md b/.agents/skills/openshell-cli/SKILL.md index 7e82488d06..340258961e 100644 --- a/.agents/skills/openshell-cli/SKILL.md +++ b/.agents/skills/openshell-cli/SKILL.md @@ -337,7 +337,7 @@ This is the most important multi-step workflow. It enables a tight feedback cycl An endpoint with omitted `protocol` retains explicit-proxy behavior. Explicit `protocol: tcp` requests policy DNS and transparent TCP and currently requires -the Docker runtime; unsupported runtimes reject the policy before starting the +the Docker or Podman runtime; unsupported runtimes reject the policy before starting the workload rather than activating only part of the network contract. ``` diff --git a/architecture/compute-runtimes.md b/architecture/compute-runtimes.md index f74b636120..5294f4f066 100644 --- a/architecture/compute-runtimes.md +++ b/architecture/compute-runtimes.md @@ -140,7 +140,7 @@ delete, reconciliation removes the row; otherwise it can remain `Deleting`. | Runtime | Best fit | Sandbox boundary | Notes | |---|---|---|---| | Docker | Local development with Docker available. | Container plus nested sandbox namespace. | Uses host networking so loopback gateway endpoints work from the supervisor. Advertises the combined-supervisor policy-DNS and transparent-TCP substrate. | -| Podman | Rootless or single-machine deployments. | Container plus nested sandbox namespace. | Uses the Podman REST API and CDI GPU devices when available. Delivers the supervisor via OCI image volume by default; falls back to extracting the binary to a host-side cache and bind-mounting it when `userns` is configured (overlay does not support idmapped mounts). | +| Podman | Rootless or single-machine deployments. | Container plus nested sandbox namespace. | Uses the Podman REST API and CDI GPU devices when available. Delivers the supervisor via OCI image volume by default; falls back to extracting the binary to a host-side cache and bind-mounting it when `userns` is configured (overlay does not support idmapped mounts). Advertises the combined-supervisor policy-DNS and transparent-TCP substrate. | | Kubernetes | Cluster deployment through Helm. | Pod plus nested sandbox namespace. | Uses Kubernetes API objects, service accounts, secrets, PVC-backed workspace storage, and GPU resources. | | VM | Experimental microVM isolation. | Per-sandbox libkrun VM. | Managed endpoint-backed driver. The gateway spawns `openshell-driver-vm`, waits for its Unix socket, and then consumes it through the same remote `compute_driver.proto` path used by unmanaged endpoint drivers. The VM driver boots a cached bootstrap `rootfs.ext4`, prepares requested OCI images inside a bootstrap VM with `umoci`, attaches the prepared image disk read-only, and gives each sandbox a writable `overlay.ext4` for merged-root changes and runtime material. The driver persists each accepted launch request beside the overlay and restarts those VMs on driver startup without recreating the overlay. | | Extension | Out-of-tree drivers operated alongside the gateway. | Whatever boundary the driver implements. | Selected by a non-reserved custom `compute_drivers = [""]` entry with `[openshell.drivers.].socket_path`, or at launch time by pairing `--drivers ` with `--compute-driver-socket=`. Reserved built-in names such as `vm`, `docker`, `podman`, and `kubernetes` cannot be used as unmanaged socket endpoints. The gateway connects to a UDS the operator already provisioned, runs `GetCapabilities`, logs the advertised `driver_name`, and dispatches all sandbox lifecycle calls through `compute_driver.proto`. The driver process and socket lifecycle are operator-owned; the gateway does not spawn, supervise, or remove unmanaged extension drivers. The trust boundary is the socket's filesystem permissions: the operator must ensure only the gateway uid can read/write it. | @@ -165,7 +165,7 @@ advertise only the runtime mechanics they can guarantee: namespace and capability ownership, DNS/TCP capture installation, and coupled restart ordering. The shared supervisor remains the sole owner of DNS eligibility, synthetic mappings, process authorization, destination filtering, -pinned dialing, relay behavior, and OCSF decisions. Docker currently advertises +pinned dialing, relay behavior, and OCSF decisions. Docker and Podman advertise `policy-dns-transparent-tcp`; other runtimes reject explicit TCP policy until they implement and validate the same complete contract. The capability marker is driver-owned supervisor input and is removed from workload environments. diff --git a/crates/openshell-driver-podman/NETWORKING.md b/crates/openshell-driver-podman/NETWORKING.md index 843cb0907d..11f654ed97 100644 --- a/crates/openshell-driver-podman/NETWORKING.md +++ b/crates/openshell-driver-podman/NETWORKING.md @@ -317,6 +317,14 @@ The supervisor uses `nsenter --net=` rather than `ip netns exec` to avoid sysfs remount issues that arise under rootless Podman where real host `CAP_SYS_ADMIN` is unavailable. +For a policy with explicit `protocol: tcp` endpoints, this same inner namespace +also hosts policy DNS and transparent TCP capture. The supervisor answers only +policy-eligible names with epoch-scoped synthetic addresses, redirects TCP to +those synthetic ranges into its transparent listener, and leaves direct real-IP +dials subject to the terminal bypass fence. The Podman driver advertises this +substrate through its driver-owned runtime capability; sandbox image and policy +environment values cannot opt into it independently. + A tmpfs is mounted at `/run/netns` in the container spec so the supervisor can create named network namespaces. In rootless Podman this directory does not exist on the host, so a private tmpfs gives the supervisor its own writable diff --git a/crates/openshell-driver-podman/README.md b/crates/openshell-driver-podman/README.md index 67ba07944e..344f4e7b2e 100644 --- a/crates/openshell-driver-podman/README.md +++ b/crates/openshell-driver-podman/README.md @@ -227,6 +227,10 @@ Key points: - Nested netns: the supervisor creates a private `NetworkNamespace` with a veth pair. Sandbox processes enter this netns via `setns(fd, CLONE_NEWNET)` in the `pre_exec` hook, forcing ordinary traffic through the CONNECT proxy. +- Policy DNS and transparent TCP: the driver advertises the complete + `policy-dns-transparent-tcp` substrate. For explicit `protocol: tcp` + endpoints, the supervisor installs namespace-local DNS listeners, synthetic + routes, and TCP redirect rules before starting the workload. - Port publishing: the container spec still requests `host_port: 0` for the configured SSH port. The gateway SSH tunnel uses the supervisor relay rather than connecting directly to the published port. diff --git a/crates/openshell-driver-podman/src/container.rs b/crates/openshell-driver-podman/src/container.rs index 9f269c6e29..f4304fc18d 100644 --- a/crates/openshell-driver-podman/src/container.rs +++ b/crates/openshell-driver-podman/src/container.rs @@ -524,11 +524,11 @@ fn build_env( openshell_core::sandbox_env::TELEMETRY_ENABLED.into(), openshell_core::telemetry::enabled_env_value().into(), ); - // Runtime capabilities are driver-owned. Podman does not yet provide the - // policy DNS and transparent TCP substrate, so override image/user input. + // Runtime capabilities are driver-owned. Override image/user input with + // only the substrate that this driver configures for the supervisor. env.insert( openshell_core::sandbox_env::NETWORK_RUNTIME_CAPABILITIES.into(), - String::new(), + openshell_core::sandbox_env::POLICY_DNS_TRANSPARENT_TCP_CAPABILITY.into(), ); // 3. TLS client cert paths (when mTLS is enabled). These point to @@ -1976,14 +1976,14 @@ mod tests { } #[test] - fn container_spec_clears_unsupported_network_capabilities() { + fn container_spec_keeps_network_capabilities_driver_controlled() { use openshell_core::proto::compute::v1::{DriverSandboxSpec, DriverSandboxTemplate}; let mut sandbox = test_sandbox("test-id", "legit-name"); sandbox.spec = Some(DriverSandboxSpec { environment: std::collections::HashMap::from([( openshell_core::sandbox_env::NETWORK_RUNTIME_CAPABILITIES.to_string(), - openshell_core::sandbox_env::POLICY_DNS_TRANSPARENT_TCP_CAPABILITY.to_string(), + "spoofed".to_string(), )]), template: Some(DriverSandboxTemplate::default()), ..Default::default() @@ -1991,7 +1991,7 @@ mod tests { let spec = build_container_spec(&sandbox, &test_config()); assert_eq!( spec["env"][openshell_core::sandbox_env::NETWORK_RUNTIME_CAPABILITIES], - serde_json::json!("") + serde_json::json!(openshell_core::sandbox_env::POLICY_DNS_TRANSPARENT_TCP_CAPABILITY) ); } diff --git a/e2e/rust/tests/transparent_tcp.rs b/e2e/rust/tests/transparent_tcp.rs index 209ad354ad..af3cf1dd27 100644 --- a/e2e/rust/tests/transparent_tcp.rs +++ b/e2e/rust/tests/transparent_tcp.rs @@ -94,8 +94,8 @@ async fn wait_for_sandbox_logs( } #[tokio::test] -async fn docker_native_tcp_uses_policy_dns_and_fails_closed_on_wrong_port_and_real_ip() { - if !is_e2e_driver("docker") { +async fn local_container_native_tcp_uses_policy_dns_and_fails_closed() { + if !is_e2e_driver("docker") && !is_e2e_driver("podman") { return; } @@ -131,7 +131,7 @@ serve({FIXTURE_PORT}) "Ready", ) .await - .expect("create Docker sandbox"); + .expect("create local-container sandbox"); let script = format!( r#"import os, socket From 507e005f27523db361d80ab6741b4182dcd1f2d4 Mon Sep 17 00:00:00 2001 From: John Myers <9696606+johntmyers@users.noreply.github.com> Date: Thu, 13 Aug 2026 14:07:05 -0700 Subject: [PATCH 11/30] fix(podman): permit policy DNS port binding Signed-off-by: John Myers <9696606+johntmyers@users.noreply.github.com> --- crates/openshell-driver-podman/README.md | 5 +++-- crates/openshell-driver-podman/src/container.rs | 8 ++++++-- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/crates/openshell-driver-podman/README.md b/crates/openshell-driver-podman/README.md index 344f4e7b2e..189c418401 100644 --- a/crates/openshell-driver-podman/README.md +++ b/crates/openshell-driver-podman/README.md @@ -61,7 +61,7 @@ The container spec in `container.rs` sets these security-critical fields: |---|---|---| | `user` | `0:0` | The supervisor needs root inside the container for namespace creation, proxy setup, Landlock, seccomp, and filesystem preparation. | | `cap_drop` | Selected unneeded defaults | Podman's default capability set is already restricted. The driver drops capabilities the supervisor does not need. | -| `cap_add` | `SYS_ADMIN`, `NET_ADMIN`, `SYS_PTRACE`, `SYSLOG`, `DAC_READ_SEARCH`, `SETPCAP` | Grants supervisor-only capabilities required for namespace setup, process identity, bypass diagnostics, and child bounding-set cleanup. | +| `cap_add` | `SYS_ADMIN`, `NET_ADMIN`, `NET_BIND_SERVICE`, `SYS_PTRACE`, `SYSLOG`, `DAC_READ_SEARCH`, `SETPCAP` | Grants supervisor-only capabilities required for namespace setup, policy DNS on port 53, process identity, bypass diagnostics, and child bounding-set cleanup. | | `no_new_privileges` | `true` | Prevents privilege escalation after exec. | | `seccomp_profile_path` | `unconfined` | The supervisor installs its own policy-aware BPF filter. A container-level profile can block Landlock/seccomp syscalls during setup. | | `mounts` | Private tmpfs at `/run/netns` | Lets the supervisor create named network namespaces in rootless Podman. | @@ -116,6 +116,7 @@ openshell sandbox create \ |---|---| | `SYS_ADMIN` | seccomp filter installation, namespace creation, and Landlock setup. | | `NET_ADMIN` | Network namespace veth setup, IP address assignment, routes, and nftables. | +| `NET_BIND_SERVICE` | Binding policy DNS TCP and UDP listeners to port 53 inside the nested namespace. | | `SYS_PTRACE` | Reading `/proc//exe` and walking process ancestry for binary identity. | | `SYSLOG` | Reading `/dev/kmsg` for bypass-detection diagnostics. | | `DAC_READ_SEARCH` | Reading `/proc//fd/` across UIDs so the proxy can resolve the binary responsible for a connection. | @@ -126,7 +127,7 @@ and `FOWNER` capabilities because the supervisor needs them to drop privileges and prepare writable sandbox directories. It also keeps `SETPCAP` until child setup so `drop_privileges()` can clear the child capability bounding set before exec. It drops unneeded defaults such as -`DAC_OVERRIDE`, `FSETID`, `KILL`, `NET_BIND_SERVICE`, `NET_RAW`, `SETFCAP`, +`DAC_OVERRIDE`, `FSETID`, `KILL`, `NET_RAW`, `SETFCAP`, and `SYS_CHROOT`. ## Supervisor Sideloading diff --git a/crates/openshell-driver-podman/src/container.rs b/crates/openshell-driver-podman/src/container.rs index f4304fc18d..92aa91b4d8 100644 --- a/crates/openshell-driver-podman/src/container.rs +++ b/crates/openshell-driver-podman/src/container.rs @@ -1094,8 +1094,6 @@ pub fn build_container_spec_for_image( "FSETID".into(), // Not needed: the supervisor does not send signals to arbitrary processes. "KILL".into(), - // Not needed: the supervisor does not bind privileged ports (<1024). - "NET_BIND_SERVICE".into(), // Not in Podman's default set but explicitly denied in case the image // or runtime adds it; raw sockets are not required. "NET_RAW".into(), @@ -1109,6 +1107,8 @@ pub fn build_container_spec_for_image( "SYS_ADMIN".into(), // Network namespace veth setup, IP/route configuration. "NET_ADMIN".into(), + // Policy DNS binds TCP and UDP port 53 inside the nested namespace. + "NET_BIND_SERVICE".into(), // Reading /proc//exe and ancestor walk for process identity in policy. "SYS_PTRACE".into(), // Reading /dev/kmsg for bypass-detection diagnostics. @@ -1793,6 +1793,10 @@ mod tests { .collect(); assert!(added.contains(&"SYS_ADMIN"), "missing SYS_ADMIN"); assert!(added.contains(&"NET_ADMIN"), "missing NET_ADMIN"); + assert!( + added.contains(&"NET_BIND_SERVICE"), + "missing NET_BIND_SERVICE for policy DNS port 53" + ); assert!(added.contains(&"SYS_PTRACE"), "missing SYS_PTRACE"); assert!(added.contains(&"SYSLOG"), "missing SYSLOG"); assert!( From 675df8111c6bee159b5a73b5c322c6c47f37f818 Mon Sep 17 00:00:00 2001 From: John Myers <9696606+johntmyers@users.noreply.github.com> Date: Thu, 13 Aug 2026 14:47:46 -0700 Subject: [PATCH 12/30] test(e2e): use qualified transparent TCP hostname Signed-off-by: John Myers <9696606+johntmyers@users.noreply.github.com> --- e2e/rust/tests/transparent_tcp.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/e2e/rust/tests/transparent_tcp.rs b/e2e/rust/tests/transparent_tcp.rs index af3cf1dd27..d4522e6994 100644 --- a/e2e/rust/tests/transparent_tcp.rs +++ b/e2e/rust/tests/transparent_tcp.rs @@ -11,7 +11,9 @@ use openshell_e2e::harness::container::{SupportContainer, is_e2e_driver}; use openshell_e2e::harness::sandbox::SandboxGuard; use tempfile::NamedTempFile; -const FIXTURE_ALIAS: &str = "transparent-tcp-fixture"; +// Use a qualified policy hostname so runtime-provided resolver search domains +// (for example Podman's `dns.podman`) cannot rewrite the policy identity. +const FIXTURE_ALIAS: &str = "transparent-tcp-fixture.openshell.test"; const FIXTURE_PORT: u16 = 5432; const TRANSPARENT_LISTENER_PORT: u16 = 15001; From e37c2eaf7a2c5daadf5b5ad767a012c4f286fcfe Mon Sep 17 00:00:00 2001 From: John Myers <9696606+johntmyers@users.noreply.github.com> Date: Thu, 13 Aug 2026 15:28:09 -0700 Subject: [PATCH 13/30] fix(podman): preserve exact policy DNS names Signed-off-by: John Myers <9696606+johntmyers@users.noreply.github.com> --- crates/openshell-driver-podman/NETWORKING.md | 5 +++++ crates/openshell-driver-podman/README.md | 4 +++- crates/openshell-driver-podman/src/container.rs | 8 ++++++++ 3 files changed, 16 insertions(+), 1 deletion(-) diff --git a/crates/openshell-driver-podman/NETWORKING.md b/crates/openshell-driver-podman/NETWORKING.md index 11f654ed97..ba04b0e26c 100644 --- a/crates/openshell-driver-podman/NETWORKING.md +++ b/crates/openshell-driver-podman/NETWORKING.md @@ -325,6 +325,11 @@ dials subject to the terminal bypass fence. The Podman driver advertises this substrate through its driver-owned runtime capability; sandbox image and policy environment values cannot opt into it independently. +The container spec sets Podman's DNS search list to `.`. Podman documents this +value as disabling implicit search domains; direct bridge aliases still resolve +through aardvark-dns. This keeps libc from expanding a policy endpoint into an +unauthorized `.dns.podman` query after the exact lookup. + A tmpfs is mounted at `/run/netns` in the container spec so the supervisor can create named network namespaces. In rootless Podman this directory does not exist on the host, so a private tmpfs gives the supervisor its own writable diff --git a/crates/openshell-driver-podman/README.md b/crates/openshell-driver-podman/README.md index 189c418401..d30e36a576 100644 --- a/crates/openshell-driver-podman/README.md +++ b/crates/openshell-driver-podman/README.md @@ -231,7 +231,9 @@ Key points: - Policy DNS and transparent TCP: the driver advertises the complete `policy-dns-transparent-tcp` substrate. For explicit `protocol: tcp` endpoints, the supervisor installs namespace-local DNS listeners, synthetic - routes, and TCP redirect rules before starting the workload. + routes, and TCP redirect rules before starting the workload. The container + disables Podman's implicit DNS search suffix so policy DNS evaluates the + exact endpoint name requested by the workload. - Port publishing: the container spec still requests `host_port: 0` for the configured SSH port. The gateway SSH tunnel uses the supervisor relay rather than connecting directly to the published port. diff --git a/crates/openshell-driver-podman/src/container.rs b/crates/openshell-driver-podman/src/container.rs index 92aa91b4d8..3569f42305 100644 --- a/crates/openshell-driver-podman/src/container.rs +++ b/crates/openshell-driver-podman/src/container.rs @@ -228,6 +228,8 @@ struct ContainerSpec { /// via Podman's `host-gateway` magic so sandbox containers can reach /// the gateway server running on the host in rootless mode. hostadd: Vec, + /// Search domains written to `/etc/resolv.conf` by Podman. + dns_search: Vec, netns: NetNS, // Matches libpod's network spec format, which is `{name: {opts}}` where // empty opts is a unit struct rather than `()`. Keep as a map so JSON @@ -1223,6 +1225,11 @@ pub fn build_container_spec_for_image( // reach services on the host. `host.openshell.internal` is the driver- // neutral alias used by policies and e2e tests. hostadd: hostadd_entries(config), + // Podman's documented `.` value removes implicit search domains while + // retaining its managed nameserver and direct network-alias lookups. + // Policy DNS must evaluate the exact endpoint names authored in policy; + // suffix-expanded names otherwise produce spurious policy denials. + dns_search: vec![".".into()], netns: NetNS { nsmode: "bridge".to_string(), }, @@ -1544,6 +1551,7 @@ mod tests { ); assert_eq!(container["user"].as_str(), Some("0:0")); assert_eq!(container["image_pull_policy"].as_str(), Some("never")); + assert_eq!(container["dns_search"], serde_json::json!(["."])); assert_eq!( container["env"][openshell_core::sandbox_env::OCI_IMAGE_USER].as_str(), Some("app:staff") From 1e5f6ac5738f02642a79525741b3c29bd387e29b Mon Sep 17 00:00:00 2001 From: John Myers <9696606+johntmyers@users.noreply.github.com> Date: Thu, 13 Aug 2026 16:01:21 -0700 Subject: [PATCH 14/30] fix(podman): route policy DNS over TCP Signed-off-by: John Myers <9696606+johntmyers@users.noreply.github.com> --- crates/openshell-driver-podman/NETWORKING.md | 5 +++++ crates/openshell-driver-podman/README.md | 4 +++- crates/openshell-driver-podman/src/container.rs | 8 ++++++++ 3 files changed, 16 insertions(+), 1 deletion(-) diff --git a/crates/openshell-driver-podman/NETWORKING.md b/crates/openshell-driver-podman/NETWORKING.md index ba04b0e26c..b42f9ce4af 100644 --- a/crates/openshell-driver-podman/NETWORKING.md +++ b/crates/openshell-driver-podman/NETWORKING.md @@ -330,6 +330,11 @@ value as disabling implicit search domains; direct bridge aliases still resolve through aardvark-dns. This keeps libc from expanding a policy endpoint into an unauthorized `.dns.podman` query after the exact lookup. +The spec also sets the resolver option `use-vc`, directing libc through the +policy DNS TCP listener. Rootless Podman's nested UDP REDIRECT path delivers +queries to the supervisor but can lose the translated reply; TCP avoids that +runtime-specific return-path failure while preserving policy DNS behavior. + A tmpfs is mounted at `/run/netns` in the container spec so the supervisor can create named network namespaces. In rootless Podman this directory does not exist on the host, so a private tmpfs gives the supervisor its own writable diff --git a/crates/openshell-driver-podman/README.md b/crates/openshell-driver-podman/README.md index d30e36a576..90a5f1f939 100644 --- a/crates/openshell-driver-podman/README.md +++ b/crates/openshell-driver-podman/README.md @@ -233,7 +233,9 @@ Key points: endpoints, the supervisor installs namespace-local DNS listeners, synthetic routes, and TCP redirect rules before starting the workload. The container disables Podman's implicit DNS search suffix so policy DNS evaluates the - exact endpoint name requested by the workload. + exact endpoint name requested by the workload, and asks libc to use the + policy DNS TCP listener to avoid rootless Podman's nested UDP NAT return + path. - Port publishing: the container spec still requests `host_port: 0` for the configured SSH port. The gateway SSH tunnel uses the supervisor relay rather than connecting directly to the published port. diff --git a/crates/openshell-driver-podman/src/container.rs b/crates/openshell-driver-podman/src/container.rs index 3569f42305..1ca4383f93 100644 --- a/crates/openshell-driver-podman/src/container.rs +++ b/crates/openshell-driver-podman/src/container.rs @@ -230,6 +230,8 @@ struct ContainerSpec { hostadd: Vec, /// Search domains written to `/etc/resolv.conf` by Podman. dns_search: Vec, + /// Resolver options written to `/etc/resolv.conf` by Podman. + dns_option: Vec, netns: NetNS, // Matches libpod's network spec format, which is `{name: {opts}}` where // empty opts is a unit struct rather than `()`. Keep as a map so JSON @@ -1230,6 +1232,11 @@ pub fn build_container_spec_for_image( // Policy DNS must evaluate the exact endpoint names authored in policy; // suffix-expanded names otherwise produce spurious policy denials. dns_search: vec![".".into()], + // Rootless Podman's nested UDP REDIRECT path can receive policy DNS + // queries without delivering the translated reply to libc. Use the + // policy DNS TCP listener, which preserves the same exact-name and + // fail-closed behavior without relying on that UDP NAT return path. + dns_option: vec!["use-vc".into()], netns: NetNS { nsmode: "bridge".to_string(), }, @@ -1552,6 +1559,7 @@ mod tests { assert_eq!(container["user"].as_str(), Some("0:0")); assert_eq!(container["image_pull_policy"].as_str(), Some("never")); assert_eq!(container["dns_search"], serde_json::json!(["."])); + assert_eq!(container["dns_option"], serde_json::json!(["use-vc"])); assert_eq!( container["env"][openshell_core::sandbox_env::OCI_IMAGE_USER].as_str(), Some("app:staff") From 114b6e0c2c57968a05ced21814f706d0d733484c Mon Sep 17 00:00:00 2001 From: John Myers <9696606+johntmyers@users.noreply.github.com> Date: Thu, 13 Aug 2026 16:32:06 -0700 Subject: [PATCH 15/30] fix(dns): serve multiple TCP queries per connection Signed-off-by: John Myers <9696606+johntmyers@users.noreply.github.com> --- .../src/policy_dns/runtime.rs | 35 +++++++++++-------- 1 file changed, 20 insertions(+), 15 deletions(-) diff --git a/crates/openshell-supervisor-network/src/policy_dns/runtime.rs b/crates/openshell-supervisor-network/src/policy_dns/runtime.rs index f65dc4c74e..490a2f6baf 100644 --- a/crates/openshell-supervisor-network/src/policy_dns/runtime.rs +++ b/crates/openshell-supervisor-network/src/policy_dns/runtime.rs @@ -113,21 +113,26 @@ impl PolicyDnsRuntime { set_tcp_nodelay_best_effort(&stream); let service = service.clone(); tokio::spawn(async move { - let Ok(wire_length) = stream.read_u16().await else { - return; - }; - let length = usize::from(wire_length); - if length > MAX_DNS_MESSAGE_BYTES { - return; - } - let mut frame = Vec::with_capacity(length + 2); - frame.extend_from_slice(&wire_length.to_be_bytes()); - frame.resize(length + 2, 0); - if stream.read_exact(&mut frame[2..]).await.is_err() { - return; - } - if let Ok(response) = wire::handle_tcp_query(&service, &frame).await { - let _ = stream.write_all(&response).await; + // DNS-over-TCP connections may carry multiple sequential + // length-prefixed messages. libc commonly reuses one + // connection for A and AAAA during getaddrinfo(). + while let Ok(wire_length) = stream.read_u16().await { + let length = usize::from(wire_length); + if length > MAX_DNS_MESSAGE_BYTES { + return; + } + let mut frame = Vec::with_capacity(length + 2); + frame.extend_from_slice(&wire_length.to_be_bytes()); + frame.resize(length + 2, 0); + if stream.read_exact(&mut frame[2..]).await.is_err() { + return; + } + let Ok(response) = wire::handle_tcp_query(&service, &frame).await else { + return; + }; + if stream.write_all(&response).await.is_err() { + return; + } } }); } From df73952861d986a473c7e067e353fa96b2f3726c Mon Sep 17 00:00:00 2001 From: John Myers <9696606+johntmyers@users.noreply.github.com> Date: Fri, 14 Aug 2026 09:53:45 -0700 Subject: [PATCH 16/30] docs(network): explain native DNS and TCP egress Signed-off-by: John Myers <9696606+johntmyers@users.noreply.github.com> --- docs/reference/policy-schema.mdx | 5 +++-- docs/sandboxes/policies.mdx | 30 ++++++++++++++++++++++++++---- 2 files changed, 29 insertions(+), 6 deletions(-) diff --git a/docs/reference/policy-schema.mdx b/docs/reference/policy-schema.mdx index a81e597f59..eac64f9785 100644 --- a/docs/reference/policy-schema.mdx +++ b/docs/reference/policy-schema.mdx @@ -163,7 +163,7 @@ Each endpoint defines a reachable destination and optional inspection rules. | `host` | string | Conditional | Hostname or IP address. Required for `protocol: tcp`; transparent TCP requires a valid DNS hostname and rejects literal IPs. A non-TCP proxy endpoint may omit `host` only when `allowed_ips` supplies the destination constraint. Supports a `*` wildcard inside the first DNS label only: `*.example.com`, `**.example.com`, and intra-label patterns like `*-aiplatform.googleapis.com` are accepted; bare `*`/`**`, TLD wildcards (`*.com`), and wildcards outside the first label are rejected at load time. | | `port` | integer | Yes | TCP port number. | | `path` | string | No | Optional HTTP path glob used to select between L7 endpoints that share the same host and port. Empty means all paths. Use this when REST and GraphQL live under the same host, such as `/repos/**` and `/graphql`. | -| `protocol` | string | No | Set to `tcp` for explicit L4 TCP passthrough without payload inspection. Omitting the field has the same payload-handling behavior, but only explicit `tcp` reserves the transparent-TCP path and therefore requires a valid DNS hostname. Set to `rest` for HTTP method/path inspection, `websocket` for RFC 6455 upgrade and client text-message inspection, `graphql` for GraphQL-over-HTTP operation inspection, `mcp` for MCP Streamable HTTP request inspection, or `json-rpc` for generic JSON-RPC-over-HTTP method inspection. WebSocket endpoints can also use GraphQL operation rules for GraphQL-over-WebSocket traffic. Provider-credentialed endpoints require an inspected protocol unless `allow_uninspected_credentials` is explicitly set. | +| `protocol` | string | No | Set to `tcp` with a valid DNS hostname to allow native TCP clients through policy DNS and transparent capture without payload inspection. Omit the field for L4 passthrough through an explicit proxy, including legacy hostless `allowed_ips` endpoints. Set to `rest` for HTTP method/path inspection, `websocket` for RFC 6455 upgrade and client text-message inspection, `graphql` for GraphQL-over-HTTP operation inspection, `mcp` for MCP Streamable HTTP request inspection, or `json-rpc` for generic JSON-RPC-over-HTTP method inspection. WebSocket endpoints can also use GraphQL operation rules for GraphQL-over-WebSocket traffic. Provider-credentialed endpoints require an inspected protocol unless `allow_uninspected_credentials` is explicitly set. | | `tls` | string | No | TLS handling mode. The proxy auto-detects TLS by peeking the first bytes of each connection and terminates it for inspected HTTPS traffic, so this field is optional in most cases. Set to `skip` to disable auto-detection for edge cases such as client-certificate mTLS or non-standard protocols. Provider-credentialed endpoints reject `tls: skip` unless `allow_uninspected_credentials` is explicitly set. The values `terminate` and `passthrough` are deprecated and log a warning; they are still accepted for backward compatibility but have no effect on behavior. | | `enforcement` | string | No | `enforce` actively blocks disallowed requests. `audit` logs violations but allows traffic through. | | `access` | string | No | Access preset. One of `read-only`, `read-write`, or `full`. Mutually exclusive with `rules`. Not valid on `protocol: mcp` or `protocol: json-rpc`; MCP uses explicit rules unless `mcp.allow_all_known_mcp_methods: true` enables the endpoint method profile, and JSON-RPC always uses explicit rules. | @@ -192,7 +192,8 @@ Each endpoint defines a reachable destination and optional inspection rules. - `access` and `rules` are mutually exclusive; setting both is rejected. - `protocol: tcp` requires a valid DNS hostname. Hostless `allowed_ips`, IP-literal hosts, trailing-dot names, and malformed DNS selectors are rejected with a policy-validation error. -- `protocol: tcp` rejects L7-only fields, including `path`, `enforcement`, `access`, `rules`, `deny_rules`, request rewriting and credential signing fields, and GraphQL, JSON-RPC, or MCP options. +- `protocol: tcp` requires at least one port and rejects L7-only fields, including `path`, `enforcement`, `access`, `rules`, `deny_rules`, request rewriting and credential signing fields, and GraphQL, JSON-RPC, or MCP options. +- A sandbox runtime must support policy DNS and transparent TCP capture before it can activate a policy containing `protocol: tcp`. Docker and Podman provide this runtime support. - When `protocol` is set, at least one of `access` or `rules` is required for `rest`, `websocket`, `graphql`, and `sql`. - `mcp` and `json-rpc` reject `access` presets; use explicit `rules`. - `json-rpc` requires explicit `rules` with `allow.method`. diff --git a/docs/sandboxes/policies.mdx b/docs/sandboxes/policies.mdx index 90796fe626..473ec667f2 100644 --- a/docs/sandboxes/policies.mdx +++ b/docs/sandboxes/policies.mdx @@ -69,7 +69,7 @@ When a hot reload changes rules, the supervisor publishes a new policy generatio | `filesystem_policy` | Static | Controls which directories the agent can access on disk. Paths are split into `read_only` and `read_write` lists. Any path not listed in either list is inaccessible. Set `include_workdir: true` to automatically add the agent's working directory to `read_write`. [Landlock LSM](https://docs.kernel.org/security/landlock.html) enforces these restrictions at the kernel level. | | `landlock` | Static | Configures Landlock LSM enforcement behavior. Set `compatibility` to `best_effort` (skip individual inaccessible paths while applying remaining rules) or `hard_requirement` (fail if any path is inaccessible or the required kernel ABI is unavailable). Refer to the [Policy Schema Reference](/reference/policy-schema#landlock) for the full behavior table. | | `process` | Static | Optionally overrides the OS-level identity for the agent process. Explicit values must be `sandbox` or numeric UID/GID values from `1` through `4294967294`; root and the invalid identity sentinel are rejected. Docker and Podman may use named identities through per-field OCI `USER` fallback; Kubernetes uses its platform-selected numeric identity. The agent also runs with seccomp filters that block dangerous system calls. | -| `network_policies` | Dynamic | Controls network access for ordinary outbound traffic from the sandbox. Each block has a name, a list of endpoints (host, port, protocol, and optional rules), and a list of binaries allowed to use those endpoints.
Every outbound connection except `https://inference.local` goes through the proxy, which queries the [policy engine](/about/how-it-works#core-components) with the destination and calling binary. A connection is allowed only when both match an entry in the same policy block.
For endpoints with `protocol: rest`, the proxy auto-detects TLS and terminates it so each HTTP request can be checked against that endpoint's `rules` (method and path). For endpoints with `protocol: websocket`, the proxy validates the RFC 6455 upgrade and evaluates `GET` rules for the handshake plus either `WEBSOCKET_TEXT` rules for raw client text messages or GraphQL operation rules for GraphQL-over-WebSocket messages. Set `websocket_credential_rewrite: true` only when a WebSocket or REST compatibility endpoint must keep placeholder credentials in sandbox-owned text frames and resolve them at the OpenShell relay boundary.
Endpoints without `protocol`, or with `protocol: tcp`, allow the TCP stream through without inspecting payloads.
If no endpoint matches, the connection is denied. Configure managed inference separately through [Inference Routing](/sandboxes/inference-routing). | +| `network_policies` | Dynamic | Controls network access for ordinary outbound traffic from the sandbox. Each block has a name, a list of endpoints (host, port, protocol, and optional rules), and a list of binaries allowed to use those endpoints.
Every outbound connection except `https://inference.local` passes through the network supervisor, which queries the [policy engine](/about/how-it-works#core-components) with the destination and calling binary. A connection is allowed only when both match an entry in the same policy block.
For endpoints with `protocol: rest`, the proxy auto-detects TLS and terminates it so each HTTP request can be checked against that endpoint's `rules` (method and path). For endpoints with `protocol: websocket`, the proxy validates the RFC 6455 upgrade and evaluates `GET` rules for the handshake plus either `WEBSOCKET_TEXT` rules for raw client text messages or GraphQL operation rules for GraphQL-over-WebSocket messages. Set `websocket_credential_rewrite: true` only when a WebSocket or REST compatibility endpoint must keep placeholder credentials in sandbox-owned text frames and resolve them at the OpenShell relay boundary.
Endpoints with `protocol: tcp` allow ordinary DNS resolution and native TCP connections without inspecting payloads. Endpoints without `protocol` retain L4 passthrough through an explicit proxy.
If no endpoint matches, the connection is denied. Configure managed inference separately through [Inference Routing](/sandboxes/inference-routing). | | `network_middlewares` | Dynamic | Declares keyed HTTP and WebSocket middleware configs. After network and L7 policy admit a request or upgrade, OpenShell matches each config's host selectors independently and runs matching entries by their unique ascending `order` before credential injection. WebSocket-capable entries continue on complete client text messages. | ## Supervisor Middleware @@ -113,6 +113,28 @@ User-specified paths in your policy YAML are not pre-filtered. If you list a pat This distinction means baseline system paths degrade gracefully while user-specified paths surface configuration errors. +## Allow Native TCP Connections + +Use `protocol: tcp` when an application needs to resolve a policy-approved hostname and open a normal TCP connection without configuring an HTTP proxy. The endpoint remains L4-only, so OpenShell authorizes the hostname, port, and calling binary but does not inspect application payloads. + +```yaml showLineNumbers={false} +network_policies: + postgres: + name: postgres + endpoints: + - host: db.internal.example + port: 5432 + protocol: tcp + binaries: + - path: /usr/bin/psql +``` + +OpenShell answers DNS only for hostnames eligible under an active `protocol: tcp` endpoint. It validates upstream answers against destination and SSRF controls, returns a supervisor-owned synthetic address, and records the validated real addresses. When the application connects to the synthetic address, OpenShell recovers the hostname and port, evaluates the calling process against the current policy generation, and dials only an address pinned by that DNS result. + +DNS resolution does not authorize a connection by itself. Unknown names, wrong ports, stale mappings, disallowed destination addresses, and binaries outside the matching policy fail closed. Applications cannot inherit access by connecting directly to a real IP returned by an upstream resolver. + +Do not combine `protocol: tcp` with L7-only fields such as `path`, `enforcement`, `access`, `rules`, `deny_rules`, request rewriting, or credential signing. Docker and Podman sandboxes support policy DNS and transparent TCP capture. Other compute drivers reject policies containing `protocol: tcp` until they provide the required runtime capability. + ## Apply a Custom Policy Pass a policy YAML file when creating the sandbox: @@ -307,7 +329,7 @@ Each segment has a fixed meaning: | `host` | Yes | Destination hostname. | | `port` | Yes | Destination port, `1` through `65535`. | | `access` | No | Access preset for L7 endpoints: `read-only`, `read-write`, or `full`. Incremental updates expand presets into protocol-specific method/path rules for REST and WebSocket endpoints. | -| `protocol` | No | Endpoint mode accepted by `openshell policy update`: `tcp`, `rest`, `websocket`, or `sql`. `tcp` explicitly selects the same L4 passthrough used when this field is omitted. `sql` is audit-only and not a recommended workflow today. Full policy YAML also supports `graphql`, `mcp`, and `json-rpc`. | +| `protocol` | No | Endpoint mode accepted by `openshell policy update`: `tcp`, `rest`, `websocket`, or `sql`. Use `tcp` for native DNS and TCP without L7 inspection. `sql` is audit-only and not a recommended workflow today. Full policy YAML also supports `graphql`, `mcp`, and `json-rpc`. | | `enforcement` | No | Enforcement mode for inspected traffic: `enforce` or `audit`. | | `options` | No | Comma-separated endpoint options. Use `websocket-credential-rewrite` with `protocol: websocket` or REST compatibility endpoints that perform a WebSocket upgrade. Use `request-body-credential-rewrite` only with `protocol: rest`. | @@ -317,7 +339,7 @@ Examples: |---|---| | `pypi.org:443` | Add a plain L4 endpoint. The proxy allows the TCP stream and does not inspect HTTP requests. | | `telemetry.example.com:443::::allow-uninspected-credentials` | Explicitly allow a provider-credentialed L4 endpoint after accepting that OpenShell cannot inspect or rewrite its traffic. | -| `db.internal.example:5432::tcp` | Add an explicit L4 endpoint. The empty `access` segment is required before `tcp`. | +| `db.internal.example:5432::tcp` | Add an L4 endpoint for native DNS resolution and transparent TCP capture. The empty `access` segment is required before `tcp`. | | `api.github.com:443:read-only:rest:enforce` | Add a REST endpoint with the `read-only` preset expanded by the policy engine into GET, HEAD, and OPTIONS access. | | `api.example.com:443:read-write:rest:enforce:request-body-credential-rewrite` | Add a REST endpoint that rewrites credential placeholders in supported text request bodies. | | `realtime.example.com:443:read-write:websocket:enforce` | Add a WebSocket endpoint with the `read-write` preset expanded by the policy engine into the upgrade `GET` and client `WEBSOCKET_TEXT` access. | @@ -608,7 +630,7 @@ Allow `pip install` and `uv pip install` to reach PyPI: - { path: /usr/local/bin/uv } ``` -Endpoints without `protocol`, or with explicit `protocol: tcp`, use TCP passthrough, where the proxy allows the stream without inspecting payloads. Explicit `protocol: tcp` does not enable direct sandbox DNS or transparent TCP capture at this stage. If the stream is HTTP and TLS is auto-terminated, the proxy can still rewrite configured credential placeholders and closes keep-alive passthrough tunnels on policy reload before forwarding another request. Provider-credentialed endpoints cannot use this shape unless `allow_uninspected_credentials: true` records the exception. WebSocket text-frame policy requires an explicit `protocol: websocket` endpoint. WebSocket payload credential rewrite can also be enabled on a `protocol: rest` compatibility endpoint with `websocket_credential_rewrite: true`. REST request body credential rewrite requires an inspected `protocol: rest` endpoint with `request_body_credential_rewrite: true`. +Endpoints without `protocol` use explicit-proxy TCP passthrough, where OpenShell allows the stream without inspecting payloads. Use `protocol: tcp` when the application needs ordinary DNS resolution and native TCP connections through transparent capture. Provider-credentialed endpoints cannot use either L4 shape unless `allow_uninspected_credentials: true` records the exception. If an explicit-proxy stream is HTTP and TLS is auto-terminated, the proxy can still rewrite configured credential placeholders and closes keep-alive passthrough tunnels on policy reload before forwarding another request. WebSocket text-frame policy requires an explicit `protocol: websocket` endpoint. WebSocket payload credential rewrite can also be enabled on a `protocol: rest` compatibility endpoint with `websocket_credential_rewrite: true`. REST request body credential rewrite requires an inspected `protocol: rest` endpoint with `request_body_credential_rewrite: true`. From 0dcac58c8d29a2609ca2dfe89b11f6cf844b6ef5 Mon Sep 17 00:00:00 2001 From: John Myers <9696606+johntmyers@users.noreply.github.com> Date: Mon, 17 Aug 2026 10:37:03 -0700 Subject: [PATCH 17/30] fix(sandbox): reconcile runtime reload with upstream Signed-off-by: John Myers <9696606+johntmyers@users.noreply.github.com> --- crates/openshell-sandbox/src/lib.rs | 66 ++++++++++++++++++----------- 1 file changed, 41 insertions(+), 25 deletions(-) diff --git a/crates/openshell-sandbox/src/lib.rs b/crates/openshell-sandbox/src/lib.rs index 1e2ca12d3d..83018a414f 100644 --- a/crates/openshell-sandbox/src/lib.rs +++ b/crates/openshell-sandbox/src/lib.rs @@ -2453,14 +2453,18 @@ impl FailedRuntimeRevision { } } +struct MiddlewareReloadContext<'a> { + desired_services: &'a [openshell_core::proto::SupervisorMiddlewareService], + authentication: &'a MiddlewareAuthentication, + registry_changed: bool, + connector: &'a MiddlewareConnector, +} + async fn reload_gateway_policy_runtime( engine: &OpaEngine, policy: Option<&openshell_core::proto::SandboxPolicy>, entrypoint_pid: u32, - desired_services: &[openshell_core::proto::SupervisorMiddlewareService], - middleware_authentication: &MiddlewareAuthentication, - middleware_registry_changed: bool, - middleware_connector: &MiddlewareConnector, + middleware: MiddlewareReloadContext<'_>, transparent_tcp: TransparentTcpReloadState, ) -> std::result::Result<(), GatewayRuntimeReloadError> { if let Some(policy) = policy @@ -2482,11 +2486,13 @@ async fn reload_gateway_policy_runtime( } } match policy { - Some(policy) if middleware_registry_changed => { - let registry = - middleware_connector(desired_services.to_vec(), middleware_authentication.clone()) - .await - .map_err(GatewayRuntimeReloadError::MiddlewareRegistry)?; + Some(policy) if middleware.registry_changed => { + let registry = (middleware.connector)( + middleware.desired_services.to_vec(), + middleware.authentication.clone(), + ) + .await + .map_err(GatewayRuntimeReloadError::MiddlewareRegistry)?; engine .reload_policy_and_middleware_from_proto_with_pid(policy, entrypoint_pid, registry) .map_err(GatewayRuntimeReloadError::PolicyValidation) @@ -3763,13 +3769,15 @@ async fn run_policy_poll_loop_with_client( &ctx.opa_engine, result.policy.as_ref(), pid, - &result.supervisor_middleware_services, - &MiddlewareAuthentication { - credentials: middleware_credentials.clone(), - enabled: result.extension_authentication_enabled, + MiddlewareReloadContext { + desired_services: &result.supervisor_middleware_services, + authentication: &MiddlewareAuthentication { + credentials: middleware_credentials.clone(), + enabled: result.extension_authentication_enabled, + }, + registry_changed: middleware_registry_changed, + connector: &ctx.middleware_connector, }, - middleware_registry_changed, - &ctx.middleware_connector, ctx.transparent_tcp, ) .await; @@ -5174,10 +5182,12 @@ network_policies: &engine, Some(&proto_policy_fixture()), 0, - &[unavailable_service], - &MiddlewareAuthentication::default(), - true, - &default_middleware_connector(), + MiddlewareReloadContext { + desired_services: &[unavailable_service], + authentication: &MiddlewareAuthentication::default(), + registry_changed: true, + connector: &default_middleware_connector(), + }, TransparentTcpReloadState::default(), ) .await @@ -5208,9 +5218,12 @@ network_policies: &engine, Some(&proto_tcp_policy_fixture()), 0, - &[], - false, - &default_middleware_connector(), + MiddlewareReloadContext { + desired_services: &[], + authentication: &MiddlewareAuthentication::default(), + registry_changed: false, + connector: &default_middleware_connector(), + }, TransparentTcpReloadState { capable: true, substrate_ready: false, @@ -5246,9 +5259,12 @@ network_policies: &engine, Some(&proto_tcp_policy_fixture()), 0, - &[], - false, - &default_middleware_connector(), + MiddlewareReloadContext { + desired_services: &[], + authentication: &MiddlewareAuthentication::default(), + registry_changed: false, + connector: &default_middleware_connector(), + }, TransparentTcpReloadState::default(), ) .await From 053a19ea923336c23cbe0c9087e0a97b0ef16e15 Mon Sep 17 00:00:00 2001 From: John Myers <9696606+johntmyers@users.noreply.github.com> Date: Mon, 17 Aug 2026 12:34:26 -0700 Subject: [PATCH 18/30] fix(network): harden transparent DNS capture Signed-off-by: John Myers <9696606+johntmyers@users.noreply.github.com> --- .../src/policy_dns/runtime.rs | 44 +++++-- .../src/policy_dns/store.rs | 36 ++++++ .../src/policy_dns/wire.rs | 34 ++++- .../openshell-supervisor-network/src/proxy.rs | 21 +++- .../src/netns/mod.rs | 13 +- .../src/netns/nft_ruleset.rs | 119 ++++-------------- 6 files changed, 159 insertions(+), 108 deletions(-) diff --git a/crates/openshell-supervisor-network/src/policy_dns/runtime.rs b/crates/openshell-supervisor-network/src/policy_dns/runtime.rs index 490a2f6baf..ad6095efa9 100644 --- a/crates/openshell-supervisor-network/src/policy_dns/runtime.rs +++ b/crates/openshell-supervisor-network/src/policy_dns/runtime.rs @@ -15,10 +15,11 @@ use std::sync::Arc; use tokio::io::{AsyncReadExt, AsyncWriteExt}; use tokio::task::JoinHandle; -const IPV4_POOL_PREFIX: u8 = 25; -const IPV6_POOL_PREFIX: u8 = 120; +const IPV4_POOL_PREFIX: u8 = 23; +const IPV6_POOL_PREFIX: u8 = 119; const IPV4_EPOCH_WINDOWS: u64 = 1 << (IPV4_POOL_PREFIX - 15); -const MAX_MAPPINGS: usize = 256; +const MAX_MAPPINGS: usize = 1024; +const MAX_CONCURRENT_UDP_QUERIES: usize = 64; #[derive(Debug, Clone)] pub(crate) struct PolicyDnsRuntimeConfig { @@ -84,6 +85,8 @@ impl PolicyDnsRuntime { let address = udp.local_addr().into_diagnostic()?; let udp_service = service.clone(); + let udp = Arc::new(udp); + let udp_concurrency = Arc::new(tokio::sync::Semaphore::new(MAX_CONCURRENT_UDP_QUERIES)); let mut udp_engine_ready = engine_ready.clone(); let udp_task = tokio::spawn(async move { if udp_engine_ready.wait_for(|ready| *ready).await.is_err() { @@ -91,13 +94,26 @@ impl PolicyDnsRuntime { } let mut request = vec![0_u8; MAX_DNS_MESSAGE_BYTES + 1]; loop { + let Ok(permit) = udp_concurrency.clone().acquire_owned().await else { + break; + }; let Ok((length, peer)) = udp.recv_from(&mut request).await else { break; }; - if let Ok(response) = wire::handle_udp_query(&udp_service, &request[..length]).await - { - let _ = udp.send_to(&response, peer).await; - } + let request = request[..length].to_vec(); + let service = udp_service.clone(); + let udp = udp.clone(); + tokio::spawn(async move { + let _permit = permit; + // Docker and Podman do not currently prove usable IPv6 + // egress. Return NOERROR/NODATA for AAAA so dual-stack + // clients can fall back to the usable IPv4 path. + if let Ok(response) = + wire::handle_udp_query_with_ipv6(&service, &request, false).await + { + let _ = udp.send_to(&response, peer).await; + } + }); } }); @@ -127,7 +143,9 @@ impl PolicyDnsRuntime { if stream.read_exact(&mut frame[2..]).await.is_err() { return; } - let Ok(response) = wire::handle_tcp_query(&service, &frame).await else { + let Ok(response) = + wire::handle_tcp_query_with_ipv6(&service, &frame, false).await + else { return; }; if stream.write_all(&response).await.is_err() { @@ -218,4 +236,14 @@ mod tests { assert!(parent.contains(&address)); } } + + #[test] + fn production_pools_and_store_capacity_expand_together() { + let config = PolicyDnsRuntimeConfig::for_epoch(7).unwrap(); + let ipv4_capacity = 1_usize << (32 - config.ipv4_cidr.prefix_len()); + let ipv6_capacity = 1_usize << (128 - config.ipv6_cidr.prefix_len()); + assert_eq!(ipv4_capacity, 512); + assert_eq!(ipv6_capacity, 512); + assert_eq!(MAX_MAPPINGS, ipv4_capacity + ipv6_capacity); + } } diff --git a/crates/openshell-supervisor-network/src/policy_dns/store.rs b/crates/openshell-supervisor-network/src/policy_dns/store.rs index 6198de744c..cf3414c771 100644 --- a/crates/openshell-supervisor-network/src/policy_dns/store.rs +++ b/crates/openshell-supervisor-network/src/policy_dns/store.rs @@ -13,6 +13,7 @@ use std::collections::{BTreeMap, BTreeSet, HashSet}; use std::net::{IpAddr, Ipv4Addr, Ipv6Addr}; use std::ops::RangeInclusive; use std::sync::RwLock; +use std::sync::atomic::{AtomicBool, Ordering}; use std::time::{Duration, Instant}; use uuid::Uuid; @@ -224,6 +225,7 @@ struct StoreState { pub(crate) struct ResolvedEndpointStore { state: RwLock, config: StoreConfig, + pool_high_water_emitted: AtomicBool, } impl ResolvedEndpointStore { @@ -244,6 +246,7 @@ impl ResolvedEndpointStore { next_mapping_generation: 0, }), config, + pool_high_water_emitted: AtomicBool::new(false), } } @@ -285,6 +288,15 @@ impl ResolvedEndpointStore { let address = allocate_address(&mut state, request.family).ok_or(PublishError::PoolExhausted)?; state.allocations.insert(key, address); + let allocated = state.allocations.len(); + if allocated.saturating_mul(5) >= self.config.max_mappings.saturating_mul(4) + && !self.pool_high_water_emitted.swap(true, Ordering::Relaxed) + { + openshell_ocsf::ocsf_emit!(build_pool_high_water_event( + allocated, + self.config.max_mappings, + )); + } address }; @@ -372,6 +384,21 @@ impl ResolvedEndpointStore { } } +fn build_pool_high_water_event(allocated: usize, capacity: usize) -> openshell_ocsf::OcsfEvent { + use openshell_ocsf::{ConfigStateChangeBuilder, SeverityId, StateId, StatusId}; + + ConfigStateChangeBuilder::new(openshell_ocsf::ctx::ctx()) + .severity(SeverityId::Low) + .status(StatusId::Success) + .state(StateId::Enabled, "high_water") + .unmapped("allocated_identities", allocated) + .unmapped("mapping_capacity", capacity) + .message(format!( + "Policy DNS synthetic pool reached high water: {allocated}/{capacity} identities allocated" + )) + .build() +} + fn allocate_address(state: &mut StoreState, family: AddressFamily) -> Option { match family { AddressFamily::Ipv4 if state.next_ipv4 <= state.end_ipv4 => { @@ -556,6 +583,15 @@ mod tests { )); } + #[test] + fn pool_high_water_event_reports_capacity_without_reclaiming_addresses() { + let event = build_pool_high_water_event(4, 5); + let json = serde_json::to_value(event).unwrap(); + assert_eq!(json["unmapped"]["allocated_identities"], 4); + assert_eq!(json["unmapped"]["mapping_capacity"], 5); + assert_eq!(json["state"], "high_water"); + } + #[test] fn older_generation_cannot_replace_newer_live_mapping() { let store = store(1); diff --git a/crates/openshell-supervisor-network/src/policy_dns/wire.rs b/crates/openshell-supervisor-network/src/policy_dns/wire.rs index d422b95f54..b9b79213d4 100644 --- a/crates/openshell-supervisor-network/src/policy_dns/wire.rs +++ b/crates/openshell-supervisor-network/src/policy_dns/wire.rs @@ -23,6 +23,14 @@ pub(crate) enum WireError { pub(crate) async fn handle_udp_query( service: &PolicyDnsService, wire: &[u8], +) -> Result, WireError> { + handle_udp_query_with_ipv6(service, wire, true).await +} + +pub(crate) async fn handle_udp_query_with_ipv6( + service: &PolicyDnsService, + wire: &[u8], + ipv6_egress: bool, ) -> Result, WireError> { let fallback_id = wire .get(..2) @@ -55,6 +63,9 @@ pub(crate) async fn handle_udp_query( }; let raw_name = query.name.to_ascii(); + if family == AddressFamily::Ipv6 && !ipv6_egress { + return encode_message(response_with_code(&request, ResponseCode::NoError)); + } match service .answer_query(&raw_name, family, Instant::now()) .await @@ -99,6 +110,14 @@ pub(crate) async fn handle_udp_query( pub(crate) async fn handle_tcp_query( service: &PolicyDnsService, frame: &[u8], +) -> Result, WireError> { + handle_tcp_query_with_ipv6(service, frame, true).await +} + +pub(crate) async fn handle_tcp_query_with_ipv6( + service: &PolicyDnsService, + frame: &[u8], + ipv6_egress: bool, ) -> Result, WireError> { let declared = frame .get(..2) @@ -107,7 +126,7 @@ pub(crate) async fn handle_tcp_query( if declared > MAX_DNS_MESSAGE_BYTES || frame.len() != declared + 2 { return Err(WireError::InvalidTcpFrame); } - let response = handle_udp_query(service, &frame[2..]).await?; + let response = handle_udp_query_with_ipv6(service, &frame[2..], ipv6_egress).await?; let length = u16::try_from(response.len()).map_err(|_| WireError::Encode)?; let mut framed = Vec::with_capacity(response.len() + 2); framed.extend_from_slice(&length.to_be_bytes()); @@ -293,6 +312,19 @@ process: { run_as_user: sandbox, run_as_group: sandbox } assert!(response.answers.is_empty()); } + #[tokio::test] + async fn runtime_without_ipv6_egress_suppresses_aaaa_without_resolving() { + let service = service(); + let wire = + handle_udp_query_with_ipv6(&service, &request("db.example.", RecordType::AAAA), false) + .await + .unwrap(); + let response = Message::from_vec(&wire).unwrap(); + assert_eq!(response.metadata.response_code, ResponseCode::NoError); + assert!(response.answers.is_empty()); + assert_eq!(service.resolver.calls.load(Ordering::SeqCst), 0); + } + #[tokio::test] async fn unsupported_type_is_not_implemented_and_malformed_tcp_is_rejected() { let service = service(); diff --git a/crates/openshell-supervisor-network/src/proxy.rs b/crates/openshell-supervisor-network/src/proxy.rs index e7552a1b9a..c2f5868d02 100644 --- a/crates/openshell-supervisor-network/src/proxy.rs +++ b/crates/openshell-supervisor-network/src/proxy.rs @@ -587,6 +587,7 @@ async fn handle_transparent_tcp_connection( original, MappingLookupError::InvalidMapping, ); + emit_activity(&activity_tx, true, "transparent_tcp_mapping"); return Ok(()); }; @@ -705,7 +706,25 @@ async fn handle_transparent_tcp_connection( "upstream_proxy_validated_ip", ), Some(upstream_proxy::ConnectTarget::Hostname) => { - unreachable!("transparent TCP must bind corporate-proxy CONNECT to a validated address") + // Transparent TCP authorization is correlated to the resolver's + // validated address set. A hostname-mode CONNECT would make the + // corporate proxy resolve again and break that binding. Treat a + // future invariant regression as an audited denial, not a panic. + emit_transparent_policy_denial(&decision, workload_addr, &host, port); + emit_denial( + &denial_tx, + &host, + port, + decision + .binary + .as_ref() + .map_or("-", |path| path.to_str().unwrap_or("-")), + &decision, + "upstream proxy did not preserve the validated IP target", + "transparent-tcp", + ); + emit_activity(&activity_tx, true, "transparent_tcp_destination"); + return Ok(()); } None => (Some(upstream_socket_peer), "direct"), }; diff --git a/crates/openshell-supervisor-process/src/netns/mod.rs b/crates/openshell-supervisor-process/src/netns/mod.rs index 30411cb4df..2b4ea554ed 100644 --- a/crates/openshell-supervisor-process/src/netns/mod.rs +++ b/crates/openshell-supervisor-process/src/netns/mod.rs @@ -21,7 +21,10 @@ use uuid::Uuid; const SUBNET_PREFIX: &str = "10.200.0"; const HOST_IP_SUFFIX: u8 = 1; const SANDBOX_IP_SUFFIX: u8 = 2; -pub const POLICY_DNS_PORT: u16 = 53; +/// Unprivileged port owned by the supervisor's policy DNS service. Workload +/// queries still target the standard DNS port and nftables redirects them to +/// this listener before the bypass fence runs. +pub const POLICY_DNS_PORT: u16 = 15_053; pub const TRANSPARENT_TCP_PORT: u16 = 15_001; const IP_SEARCH_PATHS: &[&str] = &["/usr/sbin/ip", "/sbin/ip", "/usr/bin/ip", "/bin/ip"]; const NSENTER_SEARCH_PATHS: &[&str] = &[ @@ -336,7 +339,7 @@ impl NetworkNamespace { // the local transparent listener. run_ip_netns( &self.name, - &["-6", "route", "add", synthetic_ipv6_cidr, "dev", "lo"], + &["-6", "route", "replace", synthetic_ipv6_cidr, "dev", "lo"], )?; let nft_path = find_nft().ok_or_else(|| { miette::miette!( @@ -465,7 +468,11 @@ impl NetworkNamespace { if unsafe { libc::setns(ns_fd, libc::CLONE_NEWNET) } != 0 { return Err(std::io::Error::last_os_error()); } - let address: std::net::SocketAddr = format!("0.0.0.0:{POLICY_DNS_PORT}") + // Bind the exact REDIRECT destination instead of INADDR_ANY. + // For UDP this keeps replies sourced from loopback so + // conntrack can reverse the port/address translation before + // delivering them to libc in nested rootless namespaces. + let address: std::net::SocketAddr = format!("127.0.0.1:{POLICY_DNS_PORT}") .parse() .map_err(|error| { std::io::Error::other(format!("invalid DNS listener address: {error}")) diff --git a/crates/openshell-supervisor-process/src/netns/nft_ruleset.rs b/crates/openshell-supervisor-process/src/netns/nft_ruleset.rs index 742b309267..3ec7dfaa19 100644 --- a/crates/openshell-supervisor-process/src/netns/nft_ruleset.rs +++ b/crates/openshell-supervisor-process/src/netns/nft_ruleset.rs @@ -13,10 +13,7 @@ //! `ct state` without `nf_conntrack`, `log` without `nf_log`) rolls back the //! entire transaction including table/chain creation. -/// Packet mark applied only to synthetic-destination TCP before REDIRECT. -/// The filter chain uses it to distinguish legitimate redirected listener -/// traffic from a direct dial to an arbitrary real address on the same port. -const TRANSPARENT_TCP_MARK: &str = "0x4f535450"; +const DNS_DESTINATION_PORT: &str = "53"; /// A single nft command with metadata about whether it is required. pub struct NftCommand { @@ -244,9 +241,12 @@ pub fn generate_transparent_tcp_commands( "inet", "openshell_transparent", "output", + "meta", + "nfproto", + "ipv4", "udp", "dport", - &dns_port.to_string(), + DNS_DESTINATION_PORT, "redirect", "to", &format!(":{dns_port}"), @@ -260,9 +260,12 @@ pub fn generate_transparent_tcp_commands( "inet", "openshell_transparent", "output", + "meta", + "nfproto", + "ipv4", "tcp", "dport", - &dns_port.to_string(), + DNS_DESTINATION_PORT, "redirect", "to", &format!(":{dns_port}"), @@ -282,10 +285,6 @@ pub fn generate_transparent_tcp_commands( "tcp", "dport", "1-65535", - "meta", - "mark", - "set", - TRANSPARENT_TCP_MARK, "redirect", "to", &format!(":{transparent_port}"), @@ -305,72 +304,18 @@ pub fn generate_transparent_tcp_commands( "tcp", "dport", "1-65535", - "meta", - "mark", - "set", - TRANSPARENT_TCP_MARK, "redirect", "to", &format!(":{transparent_port}"), ], ), ]; - let mut bypass = generate_bypass_commands(host_ip, proxy_port, log_prefix); - let insertion = bypass - .iter() - .position(|command| { - command.args.iter().any(|arg| arg == "log") - || command.args.iter().any(|arg| arg == "reject") - }) - .unwrap_or(bypass.len()); - let rules = [ - nft_cmd( - true, - &[ - "add", - "rule", - "inet", - "openshell_bypass", - "output", - "udp", - "dport", - &dns_port.to_string(), - "accept", - ], - ), - nft_cmd( - true, - &[ - "add", - "rule", - "inet", - "openshell_bypass", - "output", - "tcp", - "dport", - &dns_port.to_string(), - "accept", - ], - ), - nft_cmd( - true, - &[ - "add", - "rule", - "inet", - "openshell_bypass", - "output", - "meta", - "mark", - TRANSPARENT_TCP_MARK, - "tcp", - "dport", - &transparent_port.to_string(), - "accept", - ], - ), - ]; - bypass.splice(insertion..insertion, rules); + let bypass = generate_bypass_commands(host_ip, proxy_port, log_prefix); + // NAT REDIRECT rewrites both DNS and synthetic TCP to loopback before the + // filter hook. The existing `oifname lo accept` is therefore the only + // filter exception required. Authorization is enforced after accept by + // SO_ORIGINAL_DST plus the synthetic-address mapping; packet marks do not + // survive as a meaningful security boundary here. cmds.extend(bypass); cmds } @@ -610,46 +555,30 @@ mod tests { let commands = generate_transparent_tcp_commands( "10.200.0.1", 3128, - 53, + 15053, 15001, "198.18.0.0/24", "fd23:6f70:656e::/48", None, ); let text = all_strs(&commands); - assert!(text.contains("udp dport 53 redirect to :53")); - assert!(text.contains("tcp dport 53 redirect to :53")); - assert!(text.contains("udp dport 53 accept")); - assert!(text.contains( - "ip daddr 198.18.0.0/24 tcp dport 1-65535 meta mark set 0x4f535450 redirect to :15001" - )); + assert!(text.contains("meta nfproto ipv4 udp dport 53 redirect to :15053")); + assert!(text.contains("meta nfproto ipv4 tcp dport 53 redirect to :15053")); + assert!(!text.contains("udp dport 53 accept")); + assert!(text.contains("ip daddr 198.18.0.0/24 tcp dport 1-65535 redirect to :15001")); assert!( - text.contains("ip6 daddr fd23:6f70:656e::/48 tcp dport 1-65535 meta mark set 0x4f535450 redirect to :15001") + text.contains("ip6 daddr fd23:6f70:656e::/48 tcp dport 1-65535 redirect to :15001") ); - assert!(text.contains("meta mark 0x4f535450 tcp dport 15001 accept")); - assert!(!commands.iter().any(|command| { - command.args.ends_with(&[ - "tcp".to_string(), - "dport".to_string(), - "15001".to_string(), - "accept".to_string(), - ]) && !command.args.windows(3).any(|window| { - window - == [ - "meta".to_string(), - "mark".to_string(), - "0x4f535450".to_string(), - ] - }) - })); + assert!(!text.contains("meta mark")); assert!(text.contains("oifname lo accept")); assert!( - text.find("ip daddr 198.18.0.0/24 tcp dport 1-65535 meta mark set 0x4f535450 redirect to :15001") + text.find("ip daddr 198.18.0.0/24 tcp dport 1-65535 redirect to :15001") .unwrap() < text .find("meta nfproto ipv4 meta l4proto tcp reject") .unwrap() ); + assert!(!text.contains("meta nfproto ipv6 udp dport 53 redirect")); } #[test] From 3df5ee81f3c0494f7430141227c85f44fb8beed2 Mon Sep 17 00:00:00 2001 From: John Myers <9696606+johntmyers@users.noreply.github.com> Date: Mon, 17 Aug 2026 12:35:03 -0700 Subject: [PATCH 19/30] fix(podman): preserve resolver behavior for native tcp Signed-off-by: John Myers <9696606+johntmyers@users.noreply.github.com> --- crates/openshell-driver-podman/NETWORKING.md | 13 ++-- crates/openshell-driver-podman/README.md | 3 +- .../openshell-driver-podman/src/container.rs | 26 +++----- e2e/rust/tests/transparent_tcp.rs | 63 ++++++++++++++++++- 4 files changed, 75 insertions(+), 30 deletions(-) diff --git a/crates/openshell-driver-podman/NETWORKING.md b/crates/openshell-driver-podman/NETWORKING.md index b42f9ce4af..567abcbfcd 100644 --- a/crates/openshell-driver-podman/NETWORKING.md +++ b/crates/openshell-driver-podman/NETWORKING.md @@ -325,15 +325,10 @@ dials subject to the terminal bypass fence. The Podman driver advertises this substrate through its driver-owned runtime capability; sandbox image and policy environment values cannot opt into it independently. -The container spec sets Podman's DNS search list to `.`. Podman documents this -value as disabling implicit search domains; direct bridge aliases still resolve -through aardvark-dns. This keeps libc from expanding a policy endpoint into an -unauthorized `.dns.podman` query after the exact lookup. - -The spec also sets the resolver option `use-vc`, directing libc through the -policy DNS TCP listener. Rootless Podman's nested UDP REDIRECT path delivers -queries to the supervisor but can lose the translated reply; TCP avoids that -runtime-specific return-path failure while preserving policy DNS behavior. +The container spec preserves Podman's resolver search domains and options. +Policy DNS captures both UDP and TCP in the inner namespace, so it does not +depend on libc honoring `use-vc` and does not change ordinary short-name +resolution for sandboxes that do not use native TCP. A tmpfs is mounted at `/run/netns` in the container spec so the supervisor can create named network namespaces. In rootless Podman this directory does not diff --git a/crates/openshell-driver-podman/README.md b/crates/openshell-driver-podman/README.md index 90a5f1f939..f34586a8ef 100644 --- a/crates/openshell-driver-podman/README.md +++ b/crates/openshell-driver-podman/README.md @@ -61,7 +61,7 @@ The container spec in `container.rs` sets these security-critical fields: |---|---|---| | `user` | `0:0` | The supervisor needs root inside the container for namespace creation, proxy setup, Landlock, seccomp, and filesystem preparation. | | `cap_drop` | Selected unneeded defaults | Podman's default capability set is already restricted. The driver drops capabilities the supervisor does not need. | -| `cap_add` | `SYS_ADMIN`, `NET_ADMIN`, `NET_BIND_SERVICE`, `SYS_PTRACE`, `SYSLOG`, `DAC_READ_SEARCH`, `SETPCAP` | Grants supervisor-only capabilities required for namespace setup, policy DNS on port 53, process identity, bypass diagnostics, and child bounding-set cleanup. | +| `cap_add` | `SYS_ADMIN`, `NET_ADMIN`, `SYS_PTRACE`, `SYSLOG`, `DAC_READ_SEARCH`, `SETPCAP` | Grants supervisor-only capabilities required for namespace setup, process identity, bypass diagnostics, and child bounding-set cleanup. Policy DNS binds an unprivileged supervisor port and does not require `NET_BIND_SERVICE`. | | `no_new_privileges` | `true` | Prevents privilege escalation after exec. | | `seccomp_profile_path` | `unconfined` | The supervisor installs its own policy-aware BPF filter. A container-level profile can block Landlock/seccomp syscalls during setup. | | `mounts` | Private tmpfs at `/run/netns` | Lets the supervisor create named network namespaces in rootless Podman. | @@ -116,7 +116,6 @@ openshell sandbox create \ |---|---| | `SYS_ADMIN` | seccomp filter installation, namespace creation, and Landlock setup. | | `NET_ADMIN` | Network namespace veth setup, IP address assignment, routes, and nftables. | -| `NET_BIND_SERVICE` | Binding policy DNS TCP and UDP listeners to port 53 inside the nested namespace. | | `SYS_PTRACE` | Reading `/proc//exe` and walking process ancestry for binary identity. | | `SYSLOG` | Reading `/dev/kmsg` for bypass-detection diagnostics. | | `DAC_READ_SEARCH` | Reading `/proc//fd/` across UIDs so the proxy can resolve the binary responsible for a connection. | diff --git a/crates/openshell-driver-podman/src/container.rs b/crates/openshell-driver-podman/src/container.rs index 1ca4383f93..346a1f54a5 100644 --- a/crates/openshell-driver-podman/src/container.rs +++ b/crates/openshell-driver-podman/src/container.rs @@ -1111,8 +1111,6 @@ pub fn build_container_spec_for_image( "SYS_ADMIN".into(), // Network namespace veth setup, IP/route configuration. "NET_ADMIN".into(), - // Policy DNS binds TCP and UDP port 53 inside the nested namespace. - "NET_BIND_SERVICE".into(), // Reading /proc//exe and ancestor walk for process identity in policy. "SYS_PTRACE".into(), // Reading /dev/kmsg for bypass-detection diagnostics. @@ -1227,16 +1225,11 @@ pub fn build_container_spec_for_image( // reach services on the host. `host.openshell.internal` is the driver- // neutral alias used by policies and e2e tests. hostadd: hostadd_entries(config), - // Podman's documented `.` value removes implicit search domains while - // retaining its managed nameserver and direct network-alias lookups. - // Policy DNS must evaluate the exact endpoint names authored in policy; - // suffix-expanded names otherwise produce spurious policy denials. - dns_search: vec![".".into()], - // Rootless Podman's nested UDP REDIRECT path can receive policy DNS - // queries without delivering the translated reply to libc. Use the - // policy DNS TCP listener, which preserves the same exact-name and - // fail-closed behavior without relying on that UDP NAT return path. - dns_option: vec!["use-vc".into()], + // Preserve Podman's resolver defaults for both policy-DNS and ordinary + // sandboxes. Namespace-local capture supports UDP and TCP, so it must + // not depend on a libc-specific option or alter short-name searches. + dns_search: Vec::new(), + dns_option: Vec::new(), netns: NetNS { nsmode: "bridge".to_string(), }, @@ -1558,8 +1551,8 @@ mod tests { ); assert_eq!(container["user"].as_str(), Some("0:0")); assert_eq!(container["image_pull_policy"].as_str(), Some("never")); - assert_eq!(container["dns_search"], serde_json::json!(["."])); - assert_eq!(container["dns_option"], serde_json::json!(["use-vc"])); + assert_eq!(container["dns_search"], serde_json::json!([])); + assert_eq!(container["dns_option"], serde_json::json!([])); assert_eq!( container["env"][openshell_core::sandbox_env::OCI_IMAGE_USER].as_str(), Some("app:staff") @@ -1809,10 +1802,7 @@ mod tests { .collect(); assert!(added.contains(&"SYS_ADMIN"), "missing SYS_ADMIN"); assert!(added.contains(&"NET_ADMIN"), "missing NET_ADMIN"); - assert!( - added.contains(&"NET_BIND_SERVICE"), - "missing NET_BIND_SERVICE for policy DNS port 53" - ); + assert!(!added.contains(&"NET_BIND_SERVICE")); assert!(added.contains(&"SYS_PTRACE"), "missing SYS_PTRACE"); assert!(added.contains(&"SYSLOG"), "missing SYSLOG"); assert!( diff --git a/e2e/rust/tests/transparent_tcp.rs b/e2e/rust/tests/transparent_tcp.rs index d4522e6994..1ae76a2a95 100644 --- a/e2e/rust/tests/transparent_tcp.rs +++ b/e2e/rust/tests/transparent_tcp.rs @@ -14,10 +14,15 @@ use tempfile::NamedTempFile; // Use a qualified policy hostname so runtime-provided resolver search domains // (for example Podman's `dns.podman`) cannot rewrite the policy identity. const FIXTURE_ALIAS: &str = "transparent-tcp-fixture.openshell.test"; +const MUSL_FIXTURE_ALIAS: &str = "transparent-tcp-musl.openshell.test"; const FIXTURE_PORT: u16 = 5432; const TRANSPARENT_LISTENER_PORT: u16 = 15001; fn write_policy() -> Result { + write_policy_for(FIXTURE_ALIAS) +} + +fn write_policy_for(host: &str) -> Result { let mut file = NamedTempFile::new().map_err(|error| format!("create policy: {error}"))?; let policy = format!( r#"version: 1 @@ -31,7 +36,7 @@ network_policies: native_database: name: native_database endpoints: - - host: {FIXTURE_ALIAS} + - host: {host} port: {FIXTURE_PORT} protocol: tcp allowed_ips: ["10.0.0.0/8", "172.0.0.0/8", "192.168.0.0/16"] @@ -46,6 +51,62 @@ network_policies: Ok(file) } +#[tokio::test] +async fn rootless_podman_musl_client_uses_udp_policy_dns() { + if !is_e2e_driver("podman") { + return; + } + + let fixture = SupportContainer::start_python( + MUSL_FIXTURE_ALIAS, + &format!( + r#"import socket +s = socket.socket() +s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) +s.bind(('0.0.0.0', {FIXTURE_PORT})) +s.listen() +while True: + c, _ = s.accept() + data = c.recv(1024) + c.sendall(b'musl-native-tcp-ok:' + data) + c.close() +"# + ), + FIXTURE_PORT, + ) + .await + .expect("start musl TCP fixture"); + + let image = tempfile::tempdir().expect("create Alpine build context"); + std::fs::write( + image.path().join("Dockerfile"), + "FROM docker.io/library/alpine:3.22\nRUN addgroup -g 1000 sandbox && adduser -D -u 1000 -G sandbox sandbox\n", + ) + .expect("write Alpine Dockerfile"); + let policy = write_policy_for(MUSL_FIXTURE_ALIAS).expect("write musl policy"); + let policy_path = policy.path().to_string_lossy().into_owned(); + let image_path = image.path().to_string_lossy().into_owned(); + let mut sandbox = SandboxGuard::create_keep_with_args( + &["--from", &image_path, "--policy", &policy_path, "--no-tty"], + &["sh", "-c", "echo Ready; sleep infinity"], + "Ready", + ) + .await + .expect("create Alpine/musl sandbox"); + + let script = format!( + "set -eu; nslookup {MUSL_FIXTURE_ALIAS} | grep -E 'Address: 198\\.1[89]\\.'; printf probe | nc -w 5 {MUSL_FIXTURE_ALIAS} {FIXTURE_PORT} | grep musl-native-tcp-ok:probe; echo musl-policy-dns-ok" + ); + let output = sandbox + .exec(&["sh", "-c", &script]) + .await + .expect("exercise musl UDP policy DNS"); + assert!(output.contains("musl-policy-dns-ok"), "{output}"); + + sandbox.cleanup().await; + drop(fixture); +} + async fn run_cli(args: &[&str]) -> Result { let output = openshell_cmd() .args(args) From e20653d8257c8b7f831c14fd2d62260c0ba0b374 Mon Sep 17 00:00:00 2001 From: John Myers <9696606+johntmyers@users.noreply.github.com> Date: Mon, 17 Aug 2026 12:35:43 -0700 Subject: [PATCH 20/30] docs(network): clarify native tcp runtime constraints Signed-off-by: John Myers <9696606+johntmyers@users.noreply.github.com> --- architecture/sandbox.md | 6 ++++++ docs/reference/policy-schema.mdx | 3 ++- docs/sandboxes/policies.mdx | 6 ++++++ 3 files changed, 14 insertions(+), 1 deletion(-) diff --git a/architecture/sandbox.md b/architecture/sandbox.md index 4059fbd1e0..99c972176d 100644 --- a/architecture/sandbox.md +++ b/architecture/sandbox.md @@ -99,6 +99,12 @@ TCP endpoints rejects a hot reload that introduces one and keeps its complete previous policy active; recreating the sandbox installs the substrate before the workload starts. A sandbox that started with the substrate may continue to remove and re-add TCP endpoints through ordinary atomic policy reloads. +Workload DNS targets port 53, while nftables redirects eligible IPv4 DNS traffic +to an unprivileged supervisor listener. The ordinary loopback accept rule admits +the redirected socket; `SO_ORIGINAL_DST`, synthetic mapping lookup, endpoint +correlation, and generation-pinned authorization form the security boundary. +Docker and Podman do not currently advertise usable IPv6 egress for this +substrate, so AAAA queries return NOERROR/NODATA and IPv6 DNS remains fenced. Provider credential placeholders are resolved through the live provider state for each HTTP request, after destination and L7 policy admission. A static diff --git a/docs/reference/policy-schema.mdx b/docs/reference/policy-schema.mdx index eac64f9785..64942a384b 100644 --- a/docs/reference/policy-schema.mdx +++ b/docs/reference/policy-schema.mdx @@ -160,7 +160,7 @@ Each endpoint defines a reachable destination and optional inspection rules. | Field | Type | Required | Description | |---|---|---|---| -| `host` | string | Conditional | Hostname or IP address. Required for `protocol: tcp`; transparent TCP requires a valid DNS hostname and rejects literal IPs. A non-TCP proxy endpoint may omit `host` only when `allowed_ips` supplies the destination constraint. Supports a `*` wildcard inside the first DNS label only: `*.example.com`, `**.example.com`, and intra-label patterns like `*-aiplatform.googleapis.com` are accepted; bare `*`/`**`, TLD wildcards (`*.com`), and wildcards outside the first label are rejected at load time. | +| `host` | string | Conditional | Hostname or IP address. Required for `protocol: tcp`; transparent TCP requires a valid DNS hostname and rejects literal IPs. A non-TCP proxy endpoint may omit `host` only when `allowed_ips` supplies the destination constraint. Supports a `*` wildcard inside the first DNS label only: `*.example.com`, `**.example.com`, and intra-label patterns like `*-aiplatform.googleapis.com` are accepted; bare `*`/`**`, TLD wildcards (`*.com`), and wildcards outside the first label are rejected at load time. Prefer exact hosts for `protocol: tcp`: a wildcard authorizes DNS queries for all matching names and can provide a DNS-label exfiltration channel. | | `port` | integer | Yes | TCP port number. | | `path` | string | No | Optional HTTP path glob used to select between L7 endpoints that share the same host and port. Empty means all paths. Use this when REST and GraphQL live under the same host, such as `/repos/**` and `/graphql`. | | `protocol` | string | No | Set to `tcp` with a valid DNS hostname to allow native TCP clients through policy DNS and transparent capture without payload inspection. Omit the field for L4 passthrough through an explicit proxy, including legacy hostless `allowed_ips` endpoints. Set to `rest` for HTTP method/path inspection, `websocket` for RFC 6455 upgrade and client text-message inspection, `graphql` for GraphQL-over-HTTP operation inspection, `mcp` for MCP Streamable HTTP request inspection, or `json-rpc` for generic JSON-RPC-over-HTTP method inspection. WebSocket endpoints can also use GraphQL operation rules for GraphQL-over-WebSocket traffic. Provider-credentialed endpoints require an inspected protocol unless `allow_uninspected_credentials` is explicitly set. | @@ -194,6 +194,7 @@ Each endpoint defines a reachable destination and optional inspection rules. - `protocol: tcp` requires a valid DNS hostname. Hostless `allowed_ips`, IP-literal hosts, trailing-dot names, and malformed DNS selectors are rejected with a policy-validation error. - `protocol: tcp` requires at least one port and rejects L7-only fields, including `path`, `enforcement`, `access`, `rules`, `deny_rules`, request rewriting and credential signing fields, and GraphQL, JSON-RPC, or MCP options. - A sandbox runtime must support policy DNS and transparent TCP capture before it can activate a policy containing `protocol: tcp`. Docker and Podman provide this runtime support. +- Adding the first `protocol: tcp` endpoint to a running sandbox that started without one is rejected atomically because its DNS and capture substrate is startup infrastructure. Recreate the sandbox with a TCP endpoint. A sandbox that started with the substrate can remove and re-add TCP endpoints dynamically. - When `protocol` is set, at least one of `access` or `rules` is required for `rest`, `websocket`, `graphql`, and `sql`. - `mcp` and `json-rpc` reject `access` presets; use explicit `rules`. - `json-rpc` requires explicit `rules` with `allow.method`. diff --git a/docs/sandboxes/policies.mdx b/docs/sandboxes/policies.mdx index 473ec667f2..ab02000fc0 100644 --- a/docs/sandboxes/policies.mdx +++ b/docs/sandboxes/policies.mdx @@ -131,10 +131,16 @@ network_policies: OpenShell answers DNS only for hostnames eligible under an active `protocol: tcp` endpoint. It validates upstream answers against destination and SSRF controls, returns a supervisor-owned synthetic address, and records the validated real addresses. When the application connects to the synthetic address, OpenShell recovers the hostname and port, evaluates the calling process against the current policy generation, and dials only an address pinned by that DNS result. +Applications must honor the returned DNS TTL and resolve the hostname again before reconnecting after that TTL expires. A client that caches the synthetic address indefinitely can receive a connection failure after the mapping expires. Docker and Podman currently advertise only IPv4 egress for this feature, so OpenShell returns an empty successful answer for AAAA queries and lets dual-stack clients use the working A record. + DNS resolution does not authorize a connection by itself. Unknown names, wrong ports, stale mappings, disallowed destination addresses, and binaries outside the matching policy fail closed. Applications cannot inherit access by connecting directly to a real IP returned by an upstream resolver. Do not combine `protocol: tcp` with L7-only fields such as `path`, `enforcement`, `access`, `rules`, `deny_rules`, request rewriting, or credential signing. Docker and Podman sandboxes support policy DNS and transparent TCP capture. Other compute drivers reject policies containing `protocol: tcp` until they provide the required runtime capability. +Prefer exact hostnames for TCP endpoints. A wildcard authorizes DNS queries for every matching name, which means a compromised process can encode data into matching DNS labels even when the lookup does not return an address. OpenShell records failed eligible lookups for operators, but logging does not remove that exfiltration channel. + +The first TCP endpoint is a deliberate exception to ordinary dynamic network policy updates. A sandbox created without any `protocol: tcp` endpoint does not install the DNS and transparent-capture substrate. A hot reload that introduces the first TCP endpoint is rejected atomically and leaves the previous policy active; recreate the sandbox with a TCP endpoint to install the substrate. A sandbox that started with TCP support can remove and later re-add TCP endpoints through normal policy reloads. + ## Apply a Custom Policy Pass a policy YAML file when creating the sandbox: From 9733bb22e1c07dc58202ca770a0901ccd83e0c7c Mon Sep 17 00:00:00 2001 From: John Myers <9696606+johntmyers@users.noreply.github.com> Date: Mon, 17 Aug 2026 12:39:57 -0700 Subject: [PATCH 21/30] fix(network): remove unused transparent tcp pin Signed-off-by: John Myers <9696606+johntmyers@users.noreply.github.com> --- .../openshell-supervisor-network/src/proxy.rs | 19 +------------------ 1 file changed, 1 insertion(+), 18 deletions(-) diff --git a/crates/openshell-supervisor-network/src/proxy.rs b/crates/openshell-supervisor-network/src/proxy.rs index c2f5868d02..f0743f0e88 100644 --- a/crates/openshell-supervisor-network/src/proxy.rs +++ b/crates/openshell-supervisor-network/src/proxy.rs @@ -574,25 +574,8 @@ async fn handle_transparent_tcp_connection( }; let host = mapping.record.normalized_name.as_str().to_string(); let port = original.port(); - let Some(pinned_ip) = mapping - .record - .contracts - .iter() - .filter(|contract| contract.port == port) - .flat_map(|contract| contract.pinned_addresses.iter().copied()) - .next() - else { - emit_transparent_mapping_denial( - workload_addr, - original, - MappingLookupError::InvalidMapping, - ); - emit_activity(&activity_tx, true, "transparent_tcp_mapping"); - return Ok(()); - }; - let connection = crate::procfs::WorkloadProxyTcpConnection::new(workload_addr, original); - let intent = EgressIntent::transparent_tcp(host.clone(), port, pinned_ip); + let intent = EgressIntent::transparent_tcp(host.clone(), port); let engine = opa_engine.clone(); let cache = identity_cache.clone(); let pid = entrypoint_pid.clone(); From ce33537a95ff90ca45238d0b65c41292bf06e8f4 Mon Sep 17 00:00:00 2001 From: John Myers <9696606+johntmyers@users.noreply.github.com> Date: Mon, 17 Aug 2026 13:17:43 -0700 Subject: [PATCH 22/30] fix(network): admit redirected transparent tcp Signed-off-by: John Myers <9696606+johntmyers@users.noreply.github.com> --- architecture/sandbox.md | 7 ++- .../src/netns/nft_ruleset.rs | 56 +++++++++++++++++-- 2 files changed, 55 insertions(+), 8 deletions(-) diff --git a/architecture/sandbox.md b/architecture/sandbox.md index 99c972176d..eacb23f44f 100644 --- a/architecture/sandbox.md +++ b/architecture/sandbox.md @@ -100,9 +100,10 @@ previous policy active; recreating the sandbox installs the substrate before the workload starts. A sandbox that started with the substrate may continue to remove and re-add TCP endpoints through ordinary atomic policy reloads. Workload DNS targets port 53, while nftables redirects eligible IPv4 DNS traffic -to an unprivileged supervisor listener. The ordinary loopback accept rule admits -the redirected socket; `SO_ORIGINAL_DST`, synthetic mapping lookup, endpoint -correlation, and generation-pinned authorization form the security boundary. +to an unprivileged supervisor listener. The filter admits only TCP that the +kernel records as DNATed to the transparent listener; `SO_ORIGINAL_DST`, +synthetic mapping lookup, endpoint correlation, and generation-pinned +authorization form the security boundary. Docker and Podman do not currently advertise usable IPv6 egress for this substrate, so AAAA queries return NOERROR/NODATA and IPv6 DNS remains fenced. diff --git a/crates/openshell-supervisor-process/src/netns/nft_ruleset.rs b/crates/openshell-supervisor-process/src/netns/nft_ruleset.rs index 3ec7dfaa19..76fe7cf89e 100644 --- a/crates/openshell-supervisor-process/src/netns/nft_ruleset.rs +++ b/crates/openshell-supervisor-process/src/netns/nft_ruleset.rs @@ -310,12 +310,41 @@ pub fn generate_transparent_tcp_commands( ], ), ]; - let bypass = generate_bypass_commands(host_ip, proxy_port, log_prefix); + let mut bypass = generate_bypass_commands(host_ip, proxy_port, log_prefix); // NAT REDIRECT rewrites both DNS and synthetic TCP to loopback before the - // filter hook. The existing `oifname lo accept` is therefore the only - // filter exception required. Authorization is enforced after accept by - // SO_ORIGINAL_DST plus the synthetic-address mapping; packet marks do not - // survive as a meaningful security boundary here. + // filter hook. Some kernels retain the packet's pre-REDIRECT output + // interface for filter matching, so `oifname lo accept` alone is not + // portable. Admit only connections that the kernel records as DNATed to + // the transparent listener. A direct dial to that port has no DNAT status + // and still reaches the terminal bypass reject. Authorization after accept + // remains bound by SO_ORIGINAL_DST plus the synthetic-address mapping. + let insertion = bypass + .iter() + .position(|command| { + command.args.iter().any(|arg| arg == "log") + || command.args.iter().any(|arg| arg == "reject") + }) + .unwrap_or(bypass.len()); + bypass.insert( + insertion, + nft_cmd( + true, + &[ + "add", + "rule", + "inet", + "openshell_bypass", + "output", + "ct", + "status", + "dnat", + "tcp", + "dport", + &transparent_port.to_string(), + "accept", + ], + ), + ); cmds.extend(bypass); cmds } @@ -570,7 +599,24 @@ mod tests { text.contains("ip6 daddr fd23:6f70:656e::/48 tcp dport 1-65535 redirect to :15001") ); assert!(!text.contains("meta mark")); + assert!(text.contains("ct status dnat tcp dport 15001 accept")); + assert!(!commands.iter().any(|command| { + command.args.ends_with(&[ + "tcp".to_string(), + "dport".to_string(), + "15001".to_string(), + "accept".to_string(), + ]) && !command.args.windows(3).any(|window| { + window == ["ct".to_string(), "status".to_string(), "dnat".to_string()] + }) + })); assert!(text.contains("oifname lo accept")); + assert!( + text.find("ct status dnat tcp dport 15001 accept").unwrap() + < text + .find("meta nfproto ipv4 meta l4proto tcp reject") + .unwrap() + ); assert!( text.find("ip daddr 198.18.0.0/24 tcp dport 1-65535 redirect to :15001") .unwrap() From 646d9af9b82070f383529f35056b58b9b7143f7f Mon Sep 17 00:00:00 2001 From: John Myers <9696606+johntmyers@users.noreply.github.com> Date: Mon, 17 Aug 2026 14:13:20 -0700 Subject: [PATCH 23/30] fix(network): restore podman transparent networking Signed-off-by: John Myers <9696606+johntmyers@users.noreply.github.com> --- architecture/sandbox.md | 9 +- .../src/netns/nft_ruleset.rs | 119 +++++++++++++----- e2e/rust/tests/transparent_tcp.rs | 31 +++-- 3 files changed, 112 insertions(+), 47 deletions(-) diff --git a/architecture/sandbox.md b/architecture/sandbox.md index eacb23f44f..c4bba6a55d 100644 --- a/architecture/sandbox.md +++ b/architecture/sandbox.md @@ -100,10 +100,11 @@ previous policy active; recreating the sandbox installs the substrate before the workload starts. A sandbox that started with the substrate may continue to remove and re-add TCP endpoints through ordinary atomic policy reloads. Workload DNS targets port 53, while nftables redirects eligible IPv4 DNS traffic -to an unprivileged supervisor listener. The filter admits only TCP that the -kernel records as DNATed to the transparent listener; `SO_ORIGINAL_DST`, -synthetic mapping lookup, endpoint correlation, and generation-pinned -authorization form the security boundary. +to an unprivileged supervisor listener. The filter admits DNS and transparent +TCP only when the kernel records the traffic as DNATed to the corresponding +supervisor listener, so direct dials to either unprivileged listener port remain +fenced. `SO_ORIGINAL_DST`, synthetic mapping lookup, endpoint correlation, and +generation-pinned authorization form the transparent TCP security boundary. Docker and Podman do not currently advertise usable IPv6 egress for this substrate, so AAAA queries return NOERROR/NODATA and IPv6 DNS remains fenced. diff --git a/crates/openshell-supervisor-process/src/netns/nft_ruleset.rs b/crates/openshell-supervisor-process/src/netns/nft_ruleset.rs index 76fe7cf89e..874c39cf4b 100644 --- a/crates/openshell-supervisor-process/src/netns/nft_ruleset.rs +++ b/crates/openshell-supervisor-process/src/netns/nft_ruleset.rs @@ -314,10 +314,11 @@ pub fn generate_transparent_tcp_commands( // NAT REDIRECT rewrites both DNS and synthetic TCP to loopback before the // filter hook. Some kernels retain the packet's pre-REDIRECT output // interface for filter matching, so `oifname lo accept` alone is not - // portable. Admit only connections that the kernel records as DNATed to - // the transparent listener. A direct dial to that port has no DNAT status - // and still reaches the terminal bypass reject. Authorization after accept - // remains bound by SO_ORIGINAL_DST plus the synthetic-address mapping. + // portable. Admit only packets that the kernel records as DNATed to the + // supervisor listeners. A direct dial to either port has no DNAT status + // and still reaches the terminal bypass reject. Transparent TCP + // authorization after accept remains bound by SO_ORIGINAL_DST plus the + // synthetic-address mapping. let insertion = bypass .iter() .position(|command| { @@ -325,25 +326,61 @@ pub fn generate_transparent_tcp_commands( || command.args.iter().any(|arg| arg == "reject") }) .unwrap_or(bypass.len()); - bypass.insert( - insertion, - nft_cmd( - true, - &[ - "add", - "rule", - "inet", - "openshell_bypass", - "output", - "ct", - "status", - "dnat", - "tcp", - "dport", - &transparent_port.to_string(), - "accept", - ], - ), + bypass.splice( + insertion..insertion, + [ + nft_cmd( + true, + &[ + "add", + "rule", + "inet", + "openshell_bypass", + "output", + "ct", + "status", + "dnat", + "udp", + "dport", + &dns_port.to_string(), + "accept", + ], + ), + nft_cmd( + true, + &[ + "add", + "rule", + "inet", + "openshell_bypass", + "output", + "ct", + "status", + "dnat", + "tcp", + "dport", + &dns_port.to_string(), + "accept", + ], + ), + nft_cmd( + true, + &[ + "add", + "rule", + "inet", + "openshell_bypass", + "output", + "ct", + "status", + "dnat", + "tcp", + "dport", + &transparent_port.to_string(), + "accept", + ], + ), + ], ); cmds.extend(bypass); cmds @@ -599,18 +636,34 @@ mod tests { text.contains("ip6 daddr fd23:6f70:656e::/48 tcp dport 1-65535 redirect to :15001") ); assert!(!text.contains("meta mark")); + assert!(text.contains("ct status dnat udp dport 15053 accept")); + assert!(text.contains("ct status dnat tcp dport 15053 accept")); assert!(text.contains("ct status dnat tcp dport 15001 accept")); - assert!(!commands.iter().any(|command| { - command.args.ends_with(&[ - "tcp".to_string(), - "dport".to_string(), - "15001".to_string(), - "accept".to_string(), - ]) && !command.args.windows(3).any(|window| { - window == ["ct".to_string(), "status".to_string(), "dnat".to_string()] - }) - })); + for (protocol, port) in [("udp", "15053"), ("tcp", "15053"), ("tcp", "15001")] { + assert!(!commands.iter().any(|command| { + command.args.ends_with(&[ + protocol.to_string(), + "dport".to_string(), + port.to_string(), + "accept".to_string(), + ]) && !command.args.windows(3).any(|window| { + window == ["ct".to_string(), "status".to_string(), "dnat".to_string()] + }) + })); + } assert!(text.contains("oifname lo accept")); + assert!( + text.find("ct status dnat tcp dport 15053 accept").unwrap() + < text + .find("meta nfproto ipv4 meta l4proto tcp reject") + .unwrap() + ); + assert!( + text.find("ct status dnat udp dport 15053 accept").unwrap() + < text + .find("meta nfproto ipv4 meta l4proto udp reject") + .unwrap() + ); assert!( text.find("ct status dnat tcp dport 15001 accept").unwrap() < text diff --git a/e2e/rust/tests/transparent_tcp.rs b/e2e/rust/tests/transparent_tcp.rs index 1ae76a2a95..e01225f95f 100644 --- a/e2e/rust/tests/transparent_tcp.rs +++ b/e2e/rust/tests/transparent_tcp.rs @@ -23,6 +23,14 @@ fn write_policy() -> Result { } fn write_policy_for(host: &str) -> Result { + write_policy_for_identity(host, "sandbox", "sandbox") +} + +fn write_policy_for_identity( + host: &str, + run_as_user: &str, + run_as_group: &str, +) -> Result { let mut file = NamedTempFile::new().map_err(|error| format!("create policy: {error}"))?; let policy = format!( r#"version: 1 @@ -31,7 +39,7 @@ filesystem_policy: read_only: [/usr, /lib, /proc, /dev/urandom, /app, /etc, /var/log] read_write: [/sandbox, /tmp, /dev/null] landlock: {{ compatibility: best_effort }} -process: {{ run_as_user: sandbox, run_as_group: sandbox }} +process: {{ run_as_user: {run_as_user}, run_as_group: {run_as_group} }} network_policies: native_database: name: native_database @@ -77,17 +85,20 @@ while True: .await .expect("start musl TCP fixture"); - let image = tempfile::tempdir().expect("create Alpine build context"); - std::fs::write( - image.path().join("Dockerfile"), - "FROM docker.io/library/alpine:3.22\nRUN addgroup -g 1000 sandbox && adduser -D -u 1000 -G sandbox sandbox\n", - ) - .expect("write Alpine Dockerfile"); - let policy = write_policy_for(MUSL_FIXTURE_ALIAS).expect("write musl policy"); + // Use the registry image directly so the Podman gateway can pull it. A + // local --from build is performed by the CLI's Docker daemon and is not + // visible in Podman's separate image store. + let policy = + write_policy_for_identity(MUSL_FIXTURE_ALIAS, "65534", "65534").expect("write musl policy"); let policy_path = policy.path().to_string_lossy().into_owned(); - let image_path = image.path().to_string_lossy().into_owned(); let mut sandbox = SandboxGuard::create_keep_with_args( - &["--from", &image_path, "--policy", &policy_path, "--no-tty"], + &[ + "--from", + "docker.io/library/alpine:3.22", + "--policy", + &policy_path, + "--no-tty", + ], &["sh", "-c", "echo Ready; sleep infinity"], "Ready", ) From 877faf2facf73673a4e99291faa5396a1fdfe222 Mon Sep 17 00:00:00 2001 From: John Myers <9696606+johntmyers@users.noreply.github.com> Date: Mon, 17 Aug 2026 14:15:26 -0700 Subject: [PATCH 24/30] test(podman): permit alpine busybox binaries Signed-off-by: John Myers <9696606+johntmyers@users.noreply.github.com> --- e2e/rust/tests/transparent_tcp.rs | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/e2e/rust/tests/transparent_tcp.rs b/e2e/rust/tests/transparent_tcp.rs index e01225f95f..18fd15121b 100644 --- a/e2e/rust/tests/transparent_tcp.rs +++ b/e2e/rust/tests/transparent_tcp.rs @@ -23,20 +23,25 @@ fn write_policy() -> Result { } fn write_policy_for(host: &str) -> Result { - write_policy_for_identity(host, "sandbox", "sandbox") + write_policy_for_identity(host, "sandbox", "sandbox", &[]) } fn write_policy_for_identity( host: &str, run_as_user: &str, run_as_group: &str, + extra_read_only: &[&str], ) -> Result { let mut file = NamedTempFile::new().map_err(|error| format!("create policy: {error}"))?; + let extra_read_only = extra_read_only + .iter() + .map(|path| format!(", {path}")) + .collect::(); let policy = format!( r#"version: 1 filesystem_policy: include_workdir: true - read_only: [/usr, /lib, /proc, /dev/urandom, /app, /etc, /var/log] + read_only: [/usr, /lib, /proc, /dev/urandom, /app, /etc, /var/log{extra_read_only}] read_write: [/sandbox, /tmp, /dev/null] landlock: {{ compatibility: best_effort }} process: {{ run_as_user: {run_as_user}, run_as_group: {run_as_group} }} @@ -88,8 +93,11 @@ while True: // Use the registry image directly so the Podman gateway can pull it. A // local --from build is performed by the CLI's Docker daemon and is not // visible in Podman's separate image store. - let policy = - write_policy_for_identity(MUSL_FIXTURE_ALIAS, "65534", "65534").expect("write musl policy"); + // Alpine does not use Debian's /bin -> /usr/bin merge. Landlock therefore + // needs the real /bin tree for /bin/sh and the BusyBox executable used by + // nslookup and nc; this fixture does not execute anything from /sbin. + let policy = write_policy_for_identity(MUSL_FIXTURE_ALIAS, "65534", "65534", &["/bin"]) + .expect("write musl policy"); let policy_path = policy.path().to_string_lossy().into_owned(); let mut sandbox = SandboxGuard::create_keep_with_args( &[ From 3cfba71bd6b9e62a400d2936f866f27a75d6d347 Mon Sep 17 00:00:00 2001 From: John Myers <9696606+johntmyers@users.noreply.github.com> Date: Mon, 17 Aug 2026 14:56:45 -0700 Subject: [PATCH 25/30] test(podman): use portable alpine keepalive Signed-off-by: John Myers <9696606+johntmyers@users.noreply.github.com> --- e2e/rust/tests/transparent_tcp.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2e/rust/tests/transparent_tcp.rs b/e2e/rust/tests/transparent_tcp.rs index 18fd15121b..11ad447331 100644 --- a/e2e/rust/tests/transparent_tcp.rs +++ b/e2e/rust/tests/transparent_tcp.rs @@ -107,7 +107,7 @@ while True: &policy_path, "--no-tty", ], - &["sh", "-c", "echo Ready; sleep infinity"], + &["sh", "-c", "echo Ready; sleep 2147483647"], "Ready", ) .await From f074d1500fd7239172a80506dc0a784a06a820c4 Mon Sep 17 00:00:00 2001 From: John Myers <9696606+johntmyers@users.noreply.github.com> Date: Mon, 17 Aug 2026 17:11:18 -0700 Subject: [PATCH 26/30] test(podman): build musl networking fixture Signed-off-by: John Myers <9696606+johntmyers@users.noreply.github.com> --- e2e/rust/tests/transparent_tcp.rs | 96 +++++++++++++++++++++++++------ 1 file changed, 80 insertions(+), 16 deletions(-) diff --git a/e2e/rust/tests/transparent_tcp.rs b/e2e/rust/tests/transparent_tcp.rs index 11ad447331..15a16a81d3 100644 --- a/e2e/rust/tests/transparent_tcp.rs +++ b/e2e/rust/tests/transparent_tcp.rs @@ -7,7 +7,7 @@ use std::io::Write; use std::process::Stdio; use openshell_e2e::harness::binary::openshell_cmd; -use openshell_e2e::harness::container::{SupportContainer, is_e2e_driver}; +use openshell_e2e::harness::container::{ContainerEngine, SupportContainer, is_e2e_driver}; use openshell_e2e::harness::sandbox::SandboxGuard; use tempfile::NamedTempFile; @@ -18,6 +18,76 @@ const MUSL_FIXTURE_ALIAS: &str = "transparent-tcp-musl.openshell.test"; const FIXTURE_PORT: u16 = 5432; const TRANSPARENT_LISTENER_PORT: u16 = 15001; +struct MuslSandboxImage { + engine: ContainerEngine, + tag: String, +} + +impl MuslSandboxImage { + fn build() -> Result { + let engine = ContainerEngine::from_env()?; + if engine.name() != "podman" { + return Err(format!( + "musl transparent TCP E2E requires podman, got {}", + engine.name() + )); + } + + let context = + tempfile::tempdir().map_err(|error| format!("create build context: {error}"))?; + let containerfile = context.path().join("Containerfile"); + std::fs::write( + &containerfile, + "FROM docker.io/library/alpine:3.22\nRUN apk add --no-cache iproute2\n", + ) + .map_err(|error| format!("write Containerfile: {error}"))?; + + let tag = format!( + "localhost/openshell-e2e-transparent-tcp-musl:{}", + std::process::id() + ); + let output = engine + .command() + .args([ + "build", + "--file", + containerfile + .to_str() + .ok_or_else(|| "Containerfile path is not UTF-8".to_string())?, + "--tag", + &tag, + context + .path() + .to_str() + .ok_or_else(|| "build context path is not UTF-8".to_string())?, + ]) + .output() + .map_err(|error| format!("build musl sandbox image: {error}"))?; + if !output.status.success() { + return Err(format!( + "podman build failed (exit {:?}):\n{}{}", + output.status.code(), + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + )); + } + + Ok(Self { engine, tag }) + } +} + +impl Drop for MuslSandboxImage { + fn drop(&mut self) { + let _ = self + .engine + .command() + .args(["image", "rm", "--force", &self.tag]) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .status(); + } +} + fn write_policy() -> Result { write_policy_for(FIXTURE_ALIAS) } @@ -70,6 +140,7 @@ async fn rootless_podman_musl_client_uses_udp_policy_dns() { return; } + let image = MuslSandboxImage::build().expect("build Alpine/musl sandbox image"); let fixture = SupportContainer::start_python( MUSL_FIXTURE_ALIAS, &format!( @@ -90,23 +161,16 @@ while True: .await .expect("start musl TCP fixture"); - // Use the registry image directly so the Podman gateway can pull it. A - // local --from build is performed by the CLI's Docker daemon and is not - // visible in Podman's separate image store. - // Alpine does not use Debian's /bin -> /usr/bin merge. Landlock therefore - // needs the real /bin tree for /bin/sh and the BusyBox executable used by - // nslookup and nc; this fixture does not execute anything from /sbin. - let policy = write_policy_for_identity(MUSL_FIXTURE_ALIAS, "65534", "65534", &["/bin"]) - .expect("write musl policy"); + // Build through Podman so the gateway sees the image in the same store. + // Plain Alpine's BusyBox `ip` lacks `ip netns`; full iproute2 is part of + // the sandbox runtime contract. Alpine also has real /bin and /sbin trees + // rather than Debian-style usr-merge symlinks. + let policy = + write_policy_for_identity(MUSL_FIXTURE_ALIAS, "65534", "65534", &["/bin", "/sbin"]) + .expect("write musl policy"); let policy_path = policy.path().to_string_lossy().into_owned(); let mut sandbox = SandboxGuard::create_keep_with_args( - &[ - "--from", - "docker.io/library/alpine:3.22", - "--policy", - &policy_path, - "--no-tty", - ], + &["--from", &image.tag, "--policy", &policy_path, "--no-tty"], &["sh", "-c", "echo Ready; sleep 2147483647"], "Ready", ) From dfadaf100343b81eb3b1bc077ee0d55bba9b3fcf Mon Sep 17 00:00:00 2001 From: John Myers <9696606+johntmyers@users.noreply.github.com> Date: Mon, 17 Aug 2026 18:10:13 -0700 Subject: [PATCH 27/30] test(podman): isolate musl DNS probe Signed-off-by: John Myers <9696606+johntmyers@users.noreply.github.com> --- e2e/rust/tests/transparent_tcp.rs | 133 ++++++++++++++---------------- e2e/support/musl-dns-probe.c | 128 ++++++++++++++++++++++++++++ 2 files changed, 192 insertions(+), 69 deletions(-) create mode 100644 e2e/support/musl-dns-probe.c diff --git a/e2e/rust/tests/transparent_tcp.rs b/e2e/rust/tests/transparent_tcp.rs index 15a16a81d3..a3be15fd9a 100644 --- a/e2e/rust/tests/transparent_tcp.rs +++ b/e2e/rust/tests/transparent_tcp.rs @@ -4,10 +4,11 @@ #![cfg(feature = "e2e")] use std::io::Write; -use std::process::Stdio; +use std::path::{Path, PathBuf}; +use std::process::{Command, Stdio}; use openshell_e2e::harness::binary::openshell_cmd; -use openshell_e2e::harness::container::{ContainerEngine, SupportContainer, is_e2e_driver}; +use openshell_e2e::harness::container::{SupportContainer, is_e2e_driver}; use openshell_e2e::harness::sandbox::SandboxGuard; use tempfile::NamedTempFile; @@ -18,73 +19,61 @@ const MUSL_FIXTURE_ALIAS: &str = "transparent-tcp-musl.openshell.test"; const FIXTURE_PORT: u16 = 5432; const TRANSPARENT_LISTENER_PORT: u16 = 15001; -struct MuslSandboxImage { - engine: ContainerEngine, - tag: String, +struct MuslDnsProbe { + _tempdir: tempfile::TempDir, + path: PathBuf, } -impl MuslSandboxImage { +impl MuslDnsProbe { fn build() -> Result { - let engine = ContainerEngine::from_env()?; - if engine.name() != "podman" { - return Err(format!( - "musl transparent TCP E2E requires podman, got {}", - engine.name() - )); - } - - let context = - tempfile::tempdir().map_err(|error| format!("create build context: {error}"))?; - let containerfile = context.path().join("Containerfile"); - std::fs::write( - &containerfile, - "FROM docker.io/library/alpine:3.22\nRUN apk add --no-cache iproute2\n", - ) - .map_err(|error| format!("write Containerfile: {error}"))?; + let target = match std::env::consts::ARCH { + "x86_64" => "x86_64-linux-musl", + "aarch64" => "aarch64-linux-musl", + arch => return Err(format!("unsupported musl DNS probe architecture: {arch}")), + }; + let tempdir = + tempfile::tempdir().map_err(|error| format!("create probe directory: {error}"))?; + let path = tempdir.path().join("musl-dns-probe"); + let source = Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../support/musl-dns-probe.c") + .canonicalize() + .map_err(|error| format!("locate musl DNS probe source: {error}"))?; - let tag = format!( - "localhost/openshell-e2e-transparent-tcp-musl:{}", - std::process::id() - ); - let output = engine - .command() + let output = Command::new("mise") .args([ - "build", - "--file", - containerfile - .to_str() - .ok_or_else(|| "Containerfile path is not UTF-8".to_string())?, - "--tag", - &tag, - context - .path() + "x", + "--", + "zig", + "cc", + "-target", + target, + "-static", + "-O2", + "-Wall", + "-Wextra", + "-Werror", + source .to_str() - .ok_or_else(|| "build context path is not UTF-8".to_string())?, + .ok_or_else(|| "probe source path is not UTF-8".to_string())?, + "-o", + path.to_str() + .ok_or_else(|| "probe output path is not UTF-8".to_string())?, ]) .output() - .map_err(|error| format!("build musl sandbox image: {error}"))?; + .map_err(|error| format!("build static musl DNS probe: {error}"))?; if !output.status.success() { return Err(format!( - "podman build failed (exit {:?}):\n{}{}", + "musl DNS probe build failed (exit {:?}):\n{}{}", output.status.code(), String::from_utf8_lossy(&output.stdout), String::from_utf8_lossy(&output.stderr) )); } - Ok(Self { engine, tag }) - } -} - -impl Drop for MuslSandboxImage { - fn drop(&mut self) { - let _ = self - .engine - .command() - .args(["image", "rm", "--force", &self.tag]) - .stdout(Stdio::null()) - .stderr(Stdio::null()) - .status(); + Ok(Self { + _tempdir: tempdir, + path, + }) } } @@ -135,12 +124,12 @@ network_policies: } #[tokio::test] -async fn rootless_podman_musl_client_uses_udp_policy_dns() { +async fn rootless_podman_musl_getaddrinfo_uses_udp_policy_dns() { if !is_e2e_driver("podman") { return; } - let image = MuslSandboxImage::build().expect("build Alpine/musl sandbox image"); + let probe = MuslDnsProbe::build().expect("build static musl DNS probe"); let fixture = SupportContainer::start_python( MUSL_FIXTURE_ALIAS, &format!( @@ -161,29 +150,35 @@ while True: .await .expect("start musl TCP fixture"); - // Build through Podman so the gateway sees the image in the same store. - // Plain Alpine's BusyBox `ip` lacks `ip netns`; full iproute2 is part of - // the sandbox runtime contract. Alpine also has real /bin and /sbin trees - // rather than Debian-style usr-merge symlinks. - let policy = - write_policy_for_identity(MUSL_FIXTURE_ALIAS, "65534", "65534", &["/bin", "/sbin"]) - .expect("write musl policy"); + // Keep sandbox-image compatibility out of this test. The statically linked + // probe executes musl's getaddrinfo inside the normal, known-good sandbox + // image, so this test isolates rootless Podman's UDP policy-DNS path. + let policy = write_policy_for(MUSL_FIXTURE_ALIAS).expect("write musl policy"); let policy_path = policy.path().to_string_lossy().into_owned(); let mut sandbox = SandboxGuard::create_keep_with_args( - &["--from", &image.tag, "--policy", &policy_path, "--no-tty"], + &["--policy", &policy_path, "--no-tty"], &["sh", "-c", "echo Ready; sleep 2147483647"], "Ready", ) .await - .expect("create Alpine/musl sandbox"); + .expect("create sandbox for musl DNS probe"); + + sandbox + .upload( + probe.path.to_str().expect("probe path is UTF-8"), + "/sandbox/musl-dns-probe", + ) + .await + .expect("upload musl DNS probe"); - let script = format!( - "set -eu; nslookup {MUSL_FIXTURE_ALIAS} | grep -E 'Address: 198\\.1[89]\\.'; printf probe | nc -w 5 {MUSL_FIXTURE_ALIAS} {FIXTURE_PORT} | grep musl-native-tcp-ok:probe; echo musl-policy-dns-ok" - ); let output = sandbox - .exec(&["sh", "-c", &script]) + .exec(&[ + "/sandbox/musl-dns-probe", + MUSL_FIXTURE_ALIAS, + &FIXTURE_PORT.to_string(), + ]) .await - .expect("exercise musl UDP policy DNS"); + .expect("exercise musl getaddrinfo over policy DNS"); assert!(output.contains("musl-policy-dns-ok"), "{output}"); sandbox.cleanup().await; diff --git a/e2e/support/musl-dns-probe.c b/e2e/support/musl-dns-probe.c new file mode 100644 index 0000000000..ce2d36e083 --- /dev/null +++ b/e2e/support/musl-dns-probe.c @@ -0,0 +1,128 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static bool is_synthetic_ipv4(const struct sockaddr *address) { + if (address->sa_family != AF_INET) { + return false; + } + + const struct sockaddr_in *ipv4 = (const struct sockaddr_in *)address; + const unsigned long host = ntohl(ipv4->sin_addr.s_addr); + return (host & 0xfffe0000UL) == 0xc6120000UL; +} + +static int connect_synthetic(const struct addrinfo *results) { + for (const struct addrinfo *candidate = results; candidate != NULL; + candidate = candidate->ai_next) { + if (!is_synthetic_ipv4(candidate->ai_addr)) { + continue; + } + + int fd = socket(candidate->ai_family, candidate->ai_socktype, + candidate->ai_protocol); + if (fd < 0) { + continue; + } + + struct timeval timeout = {.tv_sec = 5, .tv_usec = 0}; + (void)setsockopt(fd, SOL_SOCKET, SO_RCVTIMEO, &timeout, + sizeof(timeout)); + (void)setsockopt(fd, SOL_SOCKET, SO_SNDTIMEO, &timeout, + sizeof(timeout)); + + if (connect(fd, candidate->ai_addr, candidate->ai_addrlen) == 0) { + return fd; + } + (void)close(fd); + } + + return -1; +} + +static bool send_all(int fd, const char *bytes, size_t length) { + size_t sent = 0; + while (sent < length) { + const ssize_t result = send(fd, bytes + sent, length - sent, 0); + if (result <= 0) { + return false; + } + sent += (size_t)result; + } + return true; +} + +static bool receive_exact(int fd, char *bytes, size_t length) { + size_t received = 0; + while (received < length) { + const ssize_t result = recv(fd, bytes + received, length - received, 0); + if (result <= 0) { + return false; + } + received += (size_t)result; + } + return true; +} + +int main(int argc, char **argv) { + if (argc != 3) { + fprintf(stderr, "usage: %s HOST PORT\n", argv[0]); + return 2; + } + + const struct addrinfo hints = { + .ai_family = AF_UNSPEC, + .ai_socktype = SOCK_STREAM, + .ai_protocol = IPPROTO_TCP, + }; + struct addrinfo *results = NULL; + const int resolve_status = getaddrinfo(argv[1], argv[2], &hints, &results); + if (resolve_status != 0) { + fprintf(stderr, "musl getaddrinfo failed: %s\n", + gai_strerror(resolve_status)); + return 1; + } + + const int fd = connect_synthetic(results); + freeaddrinfo(results); + if (fd < 0) { + fprintf(stderr, + "musl getaddrinfo returned no connectable synthetic IPv4 address\n"); + return 1; + } + + static const char request[] = "probe"; + if (!send_all(fd, request, sizeof(request) - 1)) { + fprintf(stderr, "send failed: %s\n", strerror(errno)); + (void)close(fd); + return 1; + } + + static const char expected[] = "musl-native-tcp-ok:probe"; + char response[sizeof(expected) - 1] = {0}; + const bool received = receive_exact(fd, response, sizeof(response)); + (void)close(fd); + if (!received) { + fprintf(stderr, "receive failed: %s\n", strerror(errno)); + return 1; + } + + if (memcmp(response, expected, sizeof(response)) != 0) { + fprintf(stderr, "unexpected response: %.*s\n", (int)sizeof(response), + response); + return 1; + } + + puts("musl-policy-dns-ok"); + return 0; +} From 628f568218b811b34d50425cdb7066bc8f6f38ba Mon Sep 17 00:00:00 2001 From: John Myers <9696606+johntmyers@users.noreply.github.com> Date: Tue, 18 Aug 2026 17:20:15 -0700 Subject: [PATCH 28/30] fix(podman): keep privileged port capability dropped Signed-off-by: John Myers <9696606+johntmyers@users.noreply.github.com> --- crates/openshell-driver-podman/src/container.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/crates/openshell-driver-podman/src/container.rs b/crates/openshell-driver-podman/src/container.rs index 346a1f54a5..cc0a812c68 100644 --- a/crates/openshell-driver-podman/src/container.rs +++ b/crates/openshell-driver-podman/src/container.rs @@ -1098,6 +1098,8 @@ pub fn build_container_spec_for_image( "FSETID".into(), // Not needed: the supervisor does not send signals to arbitrary processes. "KILL".into(), + // Not needed: the supervisor does not bind privileged ports (<1024). + "NET_BIND_SERVICE".into(), // Not in Podman's default set but explicitly denied in case the image // or runtime adds it; raw sockets are not required. "NET_RAW".into(), @@ -1802,7 +1804,6 @@ mod tests { .collect(); assert!(added.contains(&"SYS_ADMIN"), "missing SYS_ADMIN"); assert!(added.contains(&"NET_ADMIN"), "missing NET_ADMIN"); - assert!(!added.contains(&"NET_BIND_SERVICE")); assert!(added.contains(&"SYS_PTRACE"), "missing SYS_PTRACE"); assert!(added.contains(&"SYSLOG"), "missing SYSLOG"); assert!( @@ -1825,6 +1826,10 @@ mod tests { .collect(); assert!(!dropped.contains(&"SETUID"), "SETUID must not be dropped"); assert!(!dropped.contains(&"SETGID"), "SETGID must not be dropped"); + assert!( + dropped.contains(&"NET_BIND_SERVICE"), + "NET_BIND_SERVICE must stay dropped; policy DNS binds an unprivileged port" + ); assert!( !dropped.contains(&"CHOWN"), "CHOWN must not be dropped (needed for prepare_filesystem chown)" From 401bdd007e022889aab4d468fc71cdbb2b6c2ba5 Mon Sep 17 00:00:00 2001 From: John Myers <9696606+johntmyers@users.noreply.github.com> Date: Wed, 19 Aug 2026 14:23:01 -0700 Subject: [PATCH 29/30] fix(network): preserve transparent TCP port 53 Signed-off-by: John Myers <9696606+johntmyers@users.noreply.github.com> --- .../src/netns/nft_ruleset.rs | 31 +++++++++++++------ e2e/rust/tests/transparent_tcp.rs | 24 ++++++++++++-- 2 files changed, 42 insertions(+), 13 deletions(-) diff --git a/crates/openshell-supervisor-process/src/netns/nft_ruleset.rs b/crates/openshell-supervisor-process/src/netns/nft_ruleset.rs index 874c39cf4b..aef95b6068 100644 --- a/crates/openshell-supervisor-process/src/netns/nft_ruleset.rs +++ b/crates/openshell-supervisor-process/src/netns/nft_ruleset.rs @@ -260,17 +260,20 @@ pub fn generate_transparent_tcp_commands( "inet", "openshell_transparent", "output", - "meta", - "nfproto", - "ipv4", + "ip", + "daddr", + synthetic_ipv4_cidr, "tcp", "dport", - DNS_DESTINATION_PORT, + "1-65535", "redirect", "to", - &format!(":{dns_port}"), + &format!(":{transparent_port}"), ], ), + // Synthetic destinations must take precedence over the generic TCP + // DNS capture. A policy endpoint may legitimately use TCP port 53; + // that connection belongs to transparent TCP, not the DNS listener. nft_cmd( true, &[ @@ -279,15 +282,15 @@ pub fn generate_transparent_tcp_commands( "inet", "openshell_transparent", "output", - "ip", - "daddr", - synthetic_ipv4_cidr, + "meta", + "nfproto", + "ipv4", "tcp", "dport", - "1-65535", + DNS_DESTINATION_PORT, "redirect", "to", - &format!(":{transparent_port}"), + &format!(":{dns_port}"), ], ), nft_cmd( @@ -678,6 +681,14 @@ mod tests { .unwrap() ); assert!(!text.contains("meta nfproto ipv6 udp dport 53 redirect")); + assert!( + text.find("ip daddr 198.18.0.0/24 tcp dport 1-65535 redirect to :15001") + .unwrap() + < text + .find("meta nfproto ipv4 tcp dport 53 redirect to :15053") + .unwrap(), + "synthetic TCP:53 must reach transparent TCP before generic DNS capture" + ); } #[test] diff --git a/e2e/rust/tests/transparent_tcp.rs b/e2e/rust/tests/transparent_tcp.rs index a3be15fd9a..e5d54049cf 100644 --- a/e2e/rust/tests/transparent_tcp.rs +++ b/e2e/rust/tests/transparent_tcp.rs @@ -17,6 +17,7 @@ use tempfile::NamedTempFile; const FIXTURE_ALIAS: &str = "transparent-tcp-fixture.openshell.test"; const MUSL_FIXTURE_ALIAS: &str = "transparent-tcp-musl.openshell.test"; const FIXTURE_PORT: u16 = 5432; +const TCP_DNS_PORT: u16 = 53; const TRANSPARENT_LISTENER_PORT: u16 = 15001; struct MuslDnsProbe { @@ -78,11 +79,17 @@ impl MuslDnsProbe { } fn write_policy() -> Result { - write_policy_for(FIXTURE_ALIAS) + write_policy_for_identity( + FIXTURE_ALIAS, + "sandbox", + "sandbox", + &[], + &[FIXTURE_PORT, TCP_DNS_PORT], + ) } fn write_policy_for(host: &str) -> Result { - write_policy_for_identity(host, "sandbox", "sandbox", &[]) + write_policy_for_identity(host, "sandbox", "sandbox", &[], &[FIXTURE_PORT]) } fn write_policy_for_identity( @@ -90,12 +97,18 @@ fn write_policy_for_identity( run_as_user: &str, run_as_group: &str, extra_read_only: &[&str], + ports: &[u16], ) -> Result { let mut file = NamedTempFile::new().map_err(|error| format!("create policy: {error}"))?; let extra_read_only = extra_read_only .iter() .map(|path| format!(", {path}")) .collect::(); + let ports = ports + .iter() + .map(u16::to_string) + .collect::>() + .join(", "); let policy = format!( r#"version: 1 filesystem_policy: @@ -109,7 +122,7 @@ network_policies: name: native_database endpoints: - host: {host} - port: {FIXTURE_PORT} + ports: [{ports}] protocol: tcp allowed_ips: ["10.0.0.0/8", "172.0.0.0/8", "192.168.0.0/16"] binaries: @@ -256,6 +269,7 @@ def serve(port): c.close() threading.Thread(target=serve, args=({TRANSPARENT_LISTENER_PORT},), daemon=True).start() +threading.Thread(target=serve, args=({TCP_DNS_PORT},), daemon=True).start() serve({FIXTURE_PORT}) "# ), @@ -289,6 +303,9 @@ assert any(ip.startswith('198.18.') or ip.startswith('198.19.') for ip in synthe with socket.create_connection(({host:?}, {port}), timeout=10) as conn: conn.sendall(b'probe') assert conn.recv(1024) == b'native-tcp-ok:probe' +with socket.create_connection(({host:?}, {tcp_dns_port}), timeout=10) as conn: + conn.sendall(b'tcp-53-probe') + assert conn.recv(1024) == b'native-tcp-ok:tcp-53-probe' def denied(host, port): try: @@ -305,6 +322,7 @@ print('transparent-tcp-e2e-ok') "#, host = FIXTURE_ALIAS, port = FIXTURE_PORT, + tcp_dns_port = TCP_DNS_PORT, wrong_port = FIXTURE_PORT + 1, real_ip = real_ip, transparent_port = TRANSPARENT_LISTENER_PORT, From 20ade8caf5c000350925983feedf11e479c2fe7b Mon Sep 17 00:00:00 2001 From: John Myers <9696606+johntmyers@users.noreply.github.com> Date: Wed, 19 Aug 2026 14:23:01 -0700 Subject: [PATCH 30/30] fix(network): report synthetic pool pressure by family Signed-off-by: John Myers <9696606+johntmyers@users.noreply.github.com> --- .../src/policy_dns/store.rs | 80 ++++++++++++++++--- 1 file changed, 71 insertions(+), 9 deletions(-) diff --git a/crates/openshell-supervisor-network/src/policy_dns/store.rs b/crates/openshell-supervisor-network/src/policy_dns/store.rs index cf3414c771..ea2cd7c0f8 100644 --- a/crates/openshell-supervisor-network/src/policy_dns/store.rs +++ b/crates/openshell-supervisor-network/src/policy_dns/store.rs @@ -143,6 +143,19 @@ impl SyntheticPools { } Ok(Self { ipv4, ipv6 }) } + + fn capacity(&self, family: AddressFamily) -> usize { + let capacity = match family { + AddressFamily::Ipv4 => { + u128::from(u32::from(*self.ipv4.end())) - u128::from(u32::from(*self.ipv4.start())) + + 1 + } + AddressFamily::Ipv6 => { + u128::from(*self.ipv6.end()) - u128::from(*self.ipv6.start()) + 1 + } + }; + usize::try_from(capacity).unwrap_or(usize::MAX) + } } #[derive(Debug, Clone)] @@ -225,7 +238,8 @@ struct StoreState { pub(crate) struct ResolvedEndpointStore { state: RwLock, config: StoreConfig, - pool_high_water_emitted: AtomicBool, + ipv4_pool_high_water_emitted: AtomicBool, + ipv6_pool_high_water_emitted: AtomicBool, } impl ResolvedEndpointStore { @@ -246,7 +260,8 @@ impl ResolvedEndpointStore { next_mapping_generation: 0, }), config, - pool_high_water_emitted: AtomicBool::new(false), + ipv4_pool_high_water_emitted: AtomicBool::new(false), + ipv6_pool_high_water_emitted: AtomicBool::new(false), } } @@ -288,13 +303,27 @@ impl ResolvedEndpointStore { let address = allocate_address(&mut state, request.family).ok_or(PublishError::PoolExhausted)?; state.allocations.insert(key, address); - let allocated = state.allocations.len(); - if allocated.saturating_mul(5) >= self.config.max_mappings.saturating_mul(4) - && !self.pool_high_water_emitted.swap(true, Ordering::Relaxed) + let allocated = state + .allocations + .keys() + .filter(|key| key.family == request.family) + .count(); + let capacity = self + .config + .pools + .capacity(request.family) + .min(self.config.max_mappings); + let emitted = match request.family { + AddressFamily::Ipv4 => &self.ipv4_pool_high_water_emitted, + AddressFamily::Ipv6 => &self.ipv6_pool_high_water_emitted, + }; + if allocated.saturating_mul(5) >= capacity.saturating_mul(4) + && !emitted.swap(true, Ordering::Relaxed) { openshell_ocsf::ocsf_emit!(build_pool_high_water_event( allocated, - self.config.max_mappings, + capacity, + request.family, )); } address @@ -384,7 +413,11 @@ impl ResolvedEndpointStore { } } -fn build_pool_high_water_event(allocated: usize, capacity: usize) -> openshell_ocsf::OcsfEvent { +fn build_pool_high_water_event( + allocated: usize, + capacity: usize, + family: AddressFamily, +) -> openshell_ocsf::OcsfEvent { use openshell_ocsf::{ConfigStateChangeBuilder, SeverityId, StateId, StatusId}; ConfigStateChangeBuilder::new(openshell_ocsf::ctx::ctx()) @@ -393,8 +426,10 @@ fn build_pool_high_water_event(allocated: usize, capacity: usize) -> openshell_o .state(StateId::Enabled, "high_water") .unmapped("allocated_identities", allocated) .unmapped("mapping_capacity", capacity) + .unmapped("address_family", family.as_str()) .message(format!( - "Policy DNS synthetic pool reached high water: {allocated}/{capacity} identities allocated" + "Policy DNS {} synthetic pool reached high water: {allocated}/{capacity} identities allocated", + family.as_str() )) .build() } @@ -585,13 +620,40 @@ mod tests { #[test] fn pool_high_water_event_reports_capacity_without_reclaiming_addresses() { - let event = build_pool_high_water_event(4, 5); + let event = build_pool_high_water_event(4, 5, AddressFamily::Ipv4); let json = serde_json::to_value(event).unwrap(); assert_eq!(json["unmapped"]["allocated_identities"], 4); assert_eq!(json["unmapped"]["mapping_capacity"], 5); + assert_eq!(json["unmapped"]["address_family"], "ipv4"); assert_eq!(json["state"], "high_water"); } + #[test] + fn production_sized_ipv4_pool_reaches_its_own_high_water_mark() { + let pools = SyntheticPools::new( + Ipv4Addr::new(198, 18, 0, 0)..=Ipv4Addr::new(198, 18, 1, 255), + "fd23:6f70:656e::".parse().unwrap()..="fd23:6f70:656e::1ff".parse().unwrap(), + ) + .unwrap(); + assert_eq!(pools.capacity(AddressFamily::Ipv4), 512); + assert_eq!(pools.capacity(AddressFamily::Ipv6), 512); + let store = ResolvedEndpointStore::new(StoreConfig::new(pools, 1024).unwrap()); + let now = Instant::now(); + + for index in 0..410 { + store + .publish( + request(&format!("host-{index}.example"), 1, Duration::from_secs(5)), + 1, + now, + ) + .unwrap(); + } + + assert!(store.ipv4_pool_high_water_emitted.load(Ordering::Relaxed)); + assert!(!store.ipv6_pool_high_water_emitted.load(Ordering::Relaxed)); + } + #[test] fn older_generation_cannot_replace_newer_live_mapping() { let store = store(1);