From 6c9eccaa21ffe2d722e7d4679ea6cb5e6f2717a9 Mon Sep 17 00:00:00 2001 From: Surma Date: Fri, 6 Nov 2026 22:31:22 +0000 Subject: [PATCH 1/2] Add WASI curl CLI and native HTTP comparisons --- userland/Cargo.lock | 10 + userland/Cargo.toml | 2 +- userland/curl/Cargo.toml | 10 + userland/curl/README.md | 83 ++++ userland/curl/src/lib.rs | 516 +++++++++++++++++++++++++ userland/curl/src/main.rs | 46 +++ userland/curl/test.nu | 93 +++++ userland/curl/tests/server/Cargo.lock | 105 +++++ userland/curl/tests/server/Cargo.toml | 7 + userland/curl/tests/server/src/main.rs | 93 +++++ 10 files changed, 964 insertions(+), 1 deletion(-) create mode 100644 userland/curl/Cargo.toml create mode 100644 userland/curl/README.md create mode 100644 userland/curl/src/lib.rs create mode 100644 userland/curl/src/main.rs create mode 100644 userland/curl/test.nu create mode 100644 userland/curl/tests/server/Cargo.lock create mode 100644 userland/curl/tests/server/Cargo.toml create mode 100644 userland/curl/tests/server/src/main.rs diff --git a/userland/Cargo.lock b/userland/Cargo.lock index c463566..8ac2d5d 100644 --- a/userland/Cargo.lock +++ b/userland/Cargo.lock @@ -744,6 +744,16 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "curl" +version = "0.1.0" +dependencies = [ + "base64", + "clap", + "url", + "wasi 0.14.7+wasi-0.2.4", +] + [[package]] name = "curve25519-dalek" version = "4.1.3" diff --git a/userland/Cargo.toml b/userland/Cargo.toml index f20996a..10f72ab 100644 --- a/userland/Cargo.toml +++ b/userland/Cargo.toml @@ -1,7 +1,7 @@ # The default CLIs of a world, built for wasm32-wasip2. Its own workspace, outside the host's. [workspace] resolver = "3" -members = ["git", "coreutils", "procps", "cwd-init/probe", "ripgrep", "diffutils", "jaq", "sed", "grep", "fd", "spawn", "findutils", "exec-tools", "ssh-client", "ssh"] +members = ["git", "coreutils", "procps", "cwd-init/probe", "ripgrep", "diffutils", "jaq", "sed", "grep", "fd", "spawn", "findutils", "exec-tools", "ssh-client", "ssh", "curl"] # WASI compatibility patches: git/vendor/PATCHES.md. [patch.crates-io] diff --git a/userland/curl/Cargo.toml b/userland/curl/Cargo.toml new file mode 100644 index 0000000..6999e9b --- /dev/null +++ b/userland/curl/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "curl" +version = "0.1.0" +edition = "2024" + +[dependencies] +clap = { version = "4", features = ["derive"] } +url = "2" +wasi = "0.14" +base64 = "0.22" diff --git a/userland/curl/README.md b/userland/curl/README.md new file mode 100644 index 0000000..e7a6f32 --- /dev/null +++ b/userland/curl/README.md @@ -0,0 +1,83 @@ +# Curl for the world + +This CLI sends HTTP and HTTPS requests through standard WASI HTTP. +The host supplies TLS and transparent credential injection. +The CLI does not request or read host credentials. +Each redirect creates a separate request, so the host checks every hop. +Verbose request output contains only headers that the CLI supplies. + +## Supported options + +- A positional HTTP or HTTPS URL. A URL without a scheme defaults to HTTP. +- `-X`, `--request` selects the method. +- Repeatable `-H`, `--header` overrides defaults or adds headers. An empty value suppresses a default header. +- `-d`, `--data` reads literal data, `@file`, or `@-`. File input removes CR, LF, and NUL bytes. +- `--data-raw` treats all input literally, including an initial `@`. +- `--data-binary` preserves file and stdin bytes. +- Data options imply POST and the form content type. Repeated data options join with `&` in argument order. +- `--json` implies POST, preserves its body, and sets JSON Content-Type and Accept headers. +- `-o`, `--output` writes to a world file. `-o -` writes to stdout. +- `-O`, `--remote-name` uses the original URL's last path segment. +- `-s`, `--silent` suppresses errors. `-S`, `--show-error` restores errors with silent mode. +- `-i`, `--include` includes response headers. `-I`, `--head` selects HEAD and prints headers. +- `-L`, `--location` follows redirects. `--max-redirs` defaults to 50. A negative value removes the limit. +- Redirects 301, 302, and 303 convert POST to GET. Redirects 307 and 308 preserve the body. +- Redirects to another origin remove explicit Authorization and Cookie headers. +- `-f`, `--fail` suppresses error bodies and returns 22 for status codes of at least 400. +- `--fail-with-body` returns 22 but retains the body. +- `-w`, `--write-out` supports `%{http_code}`, `%{url_effective}`, `%{content_type}`, `%{size_download}`, and `\n`. +- `-v`, `--verbose` prints request and response headers to stderr. +- `-u`, `--user` supplies Basic authentication. A missing password uses an empty password without a prompt. +- `-A`, `--user-agent` supplies the User-Agent header. +- `-k`, `--insecure` is accepted and ignored. The host controls TLS verification. +- `-m`, `--max-time` bounds network operations across redirects. Zero removes the deadline. +- `--help` prints the CLI's supported options. + +Unknown options return 2 with curl's unknown-option message. +HTTP failures return 22 only when a fail option applies. +Network deadlines return 28. Input errors return 26. Output errors return 23. + +## Known differences + +The CLI supports one URL per invocation. +It does not implement multipart `-F`, `--compressed`, or curl's other options. +It buffers request and response bodies. It does not show a progress meter. +The default User-Agent is `curl/8.0.0`, not the native curl version. +WASI exposes the status code, but not the HTTP version or reason phrase. +The CLI reconstructs header status lines as HTTP/1.1 with standard reason phrases for common codes. +The host can normalize response header names and order. +WASI omits hop-by-hop headers, such as Connection, from the response. +WASI rejects forbidden request headers. The CLI sets Content-Length from the body size. +Verbose output omits native curl's connection and TLS diagnostics. +Transport diagnostics describe WASI errors instead of native libcurl errors. +Argument diagnostics other than unknown options use clap's text with exit code 2. +Write-out supports only the listed substitutions and a literal format string. +The CLI does not implement curl's credential prompts, cookie jar, config file, or proxy options. + +## Validation + +Run these commands from `userland/`: + +```sh +CARGO_BUILD_JOBS=2 CARGO_TARGET_DIR=/tmp/p1-curl/target nix develop .. -c cargo build -p curl --release --locked +``` + +The shared `cwd-init/init.o` must exist first. Generate it from the repository root if needed: + +```sh +nix develop -c nu userland/cwd-init/build.nu +CARGO_BUILD_JOBS=2 CARGO_TARGET_DIR=/tmp/p1-curl/server-target nix develop -c cargo build --manifest-path userland/curl/tests/server/Cargo.toml --locked +mkdir -p /tmp/p1-curl +/tmp/p1-curl/server-target/debug/curl-test-server /tmp/p1-curl/port +``` + +Run the server as a managed job or in another terminal. +Then run the comparisons from the repository root: + +```sh +nix develop -c nu userland/curl/test.nu +``` + +The script compares deterministic local responses with native curl. +It also checks a public HTTPS GET and verifies WASI-only component imports. +The script retains its temporary files for inspection. diff --git a/userland/curl/src/lib.rs b/userland/curl/src/lib.rs new file mode 100644 index 0000000..e0e0e50 --- /dev/null +++ b/userland/curl/src/lib.rs @@ -0,0 +1,516 @@ +//! HTTP transfers through WASI, without access to host-injected credentials. +use base64::{Engine, engine::general_purpose::STANDARD}; +use clap::Parser; +use std::{ + io::{Read, Write}, + time::{Duration, Instant}, +}; +use wasi::http::{outgoing_handler, types::*}; + +#[derive(Parser, Debug)] +#[command(disable_version_flag = true)] +pub struct Args { + pub url: String, + #[arg(skip)] + pub parts: Vec<(String, u8)>, + #[arg(short = 'X', long = "request")] + pub method: Option, + #[arg(short = 'H', long = "header", allow_hyphen_values = true)] + pub headers: Vec, + #[arg(short = 'd', long = "data", allow_hyphen_values = true)] + pub data: Vec, + #[arg(long = "data-raw", allow_hyphen_values = true)] + pub raw: Vec, + #[arg(long = "data-binary", allow_hyphen_values = true)] + pub binary: Vec, + #[arg(long, allow_hyphen_values = true)] + pub json: Vec, + #[arg(short = 'o', long = "output")] + pub output: Option, + #[arg(short = 'O', long = "remote-name")] + pub remote_name: bool, + #[arg(short = 's', long = "silent")] + pub silent: bool, + #[arg(short = 'S', long = "show-error")] + pub show_error: bool, + #[arg(short = 'i', long = "include")] + pub include: bool, + #[arg(short = 'I', long = "head")] + pub head: bool, + #[arg(short = 'L', long = "location")] + pub location: bool, + #[arg(long, default_value_t = 50, allow_hyphen_values = true)] + pub max_redirs: i64, + #[arg(short = 'f', long = "fail", conflicts_with = "fail_with_body")] + pub fail: bool, + #[arg(long)] + pub fail_with_body: bool, + #[arg(short = 'w', long = "write-out")] + pub write_out: Option, + #[arg(short = 'v', long = "verbose")] + pub verbose: bool, + #[arg(short = 'u', long = "user")] + pub user: Option, + #[arg(short = 'A', long = "user-agent")] + pub agent: Option, + #[arg(short = 'k', long = "insecure")] + pub insecure: bool, + #[arg(short = 'm', long = "max-time")] + pub max_time: Option, +} +pub struct Error { + pub code: i32, + pub message: String, +} +type Result = std::result::Result; +fn error(code: i32, message: impl Into) -> Error { + Error { + code, + message: message.into(), + } +} +fn file_data(text: &str, files: bool, strip: bool) -> Result> { + let mut bytes = if files && text.starts_with('@') { + if text == "@-" { + let mut bytes = Vec::new(); + std::io::stdin() + .read_to_end(&mut bytes) + .map_err(|e| error(26, e.to_string()))?; + bytes + } else { + std::fs::read(&text[1..]).map_err(|e| error(26, e.to_string()))? + } + } else { + return Ok(text.as_bytes().to_vec()); + }; + if strip { + bytes.retain(|b| !matches!(b, b'\r' | b'\n' | 0)); + } + Ok(bytes) +} +fn wait(pollable: wasi::io::poll::Pollable, deadline: Option) -> Result<()> { + if let Some(deadline) = deadline { + let remaining = deadline + .checked_duration_since(Instant::now()) + .ok_or_else(|| error(28, "Operation timed out"))?; + let timer = wasi::clocks::monotonic_clock::subscribe_duration( + remaining.as_nanos().min(u64::MAX as u128) as u64, + ); + wasi::io::poll::poll(&[&pollable, &timer]); + if Instant::now() >= deadline { + return Err(error(28, "Operation timed out")); + } + } else { + pollable.block(); + } + Ok(()) +} +fn transport_error(e: ErrorCode) -> Error { + let code = match e { + ErrorCode::DnsError(_) | ErrorCode::DestinationNotFound => 6, + ErrorCode::ConnectionTimeout | ErrorCode::HttpResponseTimeout => 28, + ErrorCode::ConnectionRefused | ErrorCode::ConnectionTerminated => 7, + _ => 56, + }; + error(code, format!("HTTP transfer failed: {e:?}")) +} +struct Response { + status: u16, + headers: Vec<(String, Vec)>, + body: Vec, +} +fn send( + url: &url::Url, + method: &str, + headers: &[(String, Vec)], + data: Option<&[u8]>, + deadline: Option, +) -> Result { + let fields = Fields::new(); + for (name, value) in headers { + fields + .append(name, value) + .map_err(|_| error(2, "Invalid HTTP header"))?; + } + if let Some(data) = data { + fields + .set("content-length", &[data.len().to_string().into_bytes()]) + .map_err(|_| error(2, "Invalid content length"))?; + } + let request = OutgoingRequest::new(fields); + let method = match method { + "GET" => Method::Get, + "POST" => Method::Post, + "HEAD" => Method::Head, + "PUT" => Method::Put, + "DELETE" => Method::Delete, + "PATCH" => Method::Patch, + "OPTIONS" => Method::Options, + other => Method::Other(other.into()), + }; + request + .set_method(&method) + .map_err(|_| error(2, "Invalid HTTP method"))?; + request + .set_scheme(Some(&if url.scheme() == "https" { + Scheme::Https + } else { + Scheme::Http + })) + .map_err(|_| error(3, "Invalid scheme"))?; + request + .set_authority(Some( + &url[url::Position::BeforeHost..url::Position::AfterPort], + )) + .map_err(|_| error(3, "Invalid authority"))?; + request + .set_path_with_query(Some( + &url[url::Position::BeforePath..url::Position::AfterQuery], + )) + .map_err(|_| error(3, "Invalid path"))?; + let outgoing = request + .body() + .map_err(|_| error(26, "Cannot open request body"))?; + let options = RequestOptions::new(); + if let Some(deadline) = deadline { + let ns = deadline + .saturating_duration_since(Instant::now()) + .as_nanos() + .min(u64::MAX as u128) as u64; + options + .set_connect_timeout(Some(ns)) + .map_err(|_| error(28, "Timeout unsupported"))?; + options + .set_first_byte_timeout(Some(ns)) + .map_err(|_| error(28, "Timeout unsupported"))?; + options + .set_between_bytes_timeout(Some(ns)) + .map_err(|_| error(28, "Timeout unsupported"))?; + } + let future = outgoing_handler::handle(request, Some(options)).map_err(transport_error)?; + if let Some(data) = data { + let stream = outgoing + .write() + .map_err(|_| error(26, "Cannot write request"))?; + let mut remaining = data; + while !remaining.is_empty() { + wait(stream.subscribe(), deadline)?; + let count = stream + .check_write() + .map_err(|_| error(55, "Request write failed"))? as usize; + let count = count.min(remaining.len()); + stream + .write(&remaining[..count]) + .map_err(|_| error(55, "Request write failed"))?; + remaining = &remaining[count..]; + } + stream + .flush() + .map_err(|_| error(55, "Request flush failed"))?; + wait(stream.subscribe(), deadline)?; + drop(stream); + } + OutgoingBody::finish(outgoing, None).map_err(transport_error)?; + wait(future.subscribe(), deadline)?; + let response = future + .get() + .ok_or_else(|| error(56, "Missing response"))? + .map_err(|_| error(56, "Consumed response"))? + .map_err(transport_error)?; + let status = response.status(); + let headers = response.headers().entries(); + let incoming = response + .consume() + .map_err(|_| error(56, "Cannot consume response"))?; + let stream = incoming + .stream() + .map_err(|_| error(56, "Cannot read response"))?; + let mut body = Vec::new(); + loop { + wait(stream.subscribe(), deadline)?; + match stream.read(65536) { + Ok(bytes) => body.extend(bytes), + Err(wasi::io::streams::StreamError::Closed) => break, + Err(_) => return Err(error(56, "Response read failed")), + } + } + drop(stream); + let trailers = IncomingBody::finish(incoming); + wait(trailers.subscribe(), deadline)?; + trailers + .get() + .ok_or_else(|| error(56, "Missing trailers"))? + .map_err(|_| error(56, "Consumed trailers"))? + .map_err(transport_error)?; + Ok(Response { + status, + headers, + body, + }) +} +fn header_block(response: &Response) -> Vec { + let reason = match response.status { + 200 => "OK", + 201 => "Created", + 204 => "No Content", + 301 => "Moved Permanently", + 302 => "Found", + 303 => "See Other", + 307 => "Temporary Redirect", + 308 => "Permanent Redirect", + 400 => "Bad Request", + 401 => "Unauthorized", + 403 => "Forbidden", + 404 => "Not Found", + 500 => "Internal Server Error", + _ => "", + }; + let mut bytes = format!("HTTP/1.1 {} {reason}\r\n", response.status).into_bytes(); + for (name, value) in &response.headers { + bytes.extend(name.as_bytes()); + bytes.extend(b": "); + bytes.extend(value); + bytes.extend(b"\r\n"); + } + bytes.extend(b"\r\n"); + bytes +} +pub fn run(args: &Args) -> Result<()> { + let mut url = url::Url::parse(&args.url) + .or_else(|_| url::Url::parse(&format!("http://{}", args.url))) + .map_err(|_| error(3, "URL rejected"))?; + if !matches!(url.scheme(), "http" | "https") { + return Err(error(1, "Protocol not supported")); + } + let deadline = match args.max_time { + Some(n) if !n.is_finite() || n < 0.0 => return Err(error(2, "Invalid max-time")), + Some(n) if n > 0.0 => Some( + Instant::now() + .checked_add( + Duration::try_from_secs_f64(n).map_err(|_| error(2, "Invalid max-time"))?, + ) + .ok_or_else(|| error(2, "Invalid max-time"))?, + ), + _ => None, + }; + let mut data = Vec::new(); + let has_data = !args.data.is_empty() + || !args.raw.is_empty() + || !args.binary.is_empty() + || !args.json.is_empty(); + for (value, kind) in &args.parts { + if !data.is_empty() && args.json.is_empty() { + data.push(b'&'); + } + data.extend(file_data(value, *kind != 1, *kind == 0)?); + } + let mut send_data = has_data; + let mut method = if args.head { + "HEAD" + } else if has_data { + "POST" + } else { + "GET" + } + .to_string(); + if let Some(custom) = &args.method { + method.clone_from(custom); + } + let mut headers: Vec<(String, Vec)> = vec![ + ( + "User-Agent".into(), + args.agent + .as_deref() + .unwrap_or("curl/8.0.0") + .as_bytes() + .to_vec(), + ), + ( + "Accept".into(), + if args.json.is_empty() { + b"*/*".to_vec() + } else { + b"application/json".to_vec() + }, + ), + ]; + if has_data { + headers.push(( + "Content-Type".into(), + if args.json.is_empty() { + b"application/x-www-form-urlencoded".to_vec() + } else { + b"application/json".to_vec() + }, + )); + } + let user = args.user.clone().or_else(|| { + if url.username().is_empty() { + None + } else { + Some(format!( + "{}:{}", + url.username(), + url.password().unwrap_or("") + )) + } + }); + if let Some(user) = user { + let user = if user.contains(':') { + user + } else { + format!("{user}:") + }; + headers.push(( + "Authorization".into(), + format!("Basic {}", STANDARD.encode(user)).into_bytes(), + )); + } + url.set_username("").map_err(|_| error(3, "Invalid URL"))?; + url.set_password(None) + .map_err(|_| error(3, "Invalid URL"))?; + let mut custom_headers = std::collections::HashSet::new(); + for h in &args.headers { + let (name, value) = h + .split_once(':') + .or_else(|| h.strip_suffix(';').map(|name| (name, ";"))) + .ok_or_else(|| error(2, "Invalid HTTP header"))?; + if custom_headers.insert(name.to_ascii_lowercase()) { + headers.retain(|(n, _)| !n.eq_ignore_ascii_case(name)); + } + if !value.is_empty() { + headers.push(( + name.into(), + if value == ";" { + Vec::new() + } else { + value.trim_start().as_bytes().to_vec() + }, + )); + } + } + let output = args.output.clone().or_else(|| { + if args.remote_name { + url.path_segments() + .and_then(|mut s| s.next_back()) + .filter(|s| !s.is_empty()) + .map(str::to_owned) + } else { + None + } + }); + if args.remote_name && output.is_none() { + return Err(error(23, "Remote file name has no length")); + } + let mut bytes = Vec::new(); + let mut hops = 0; + let response = loop { + if args.verbose { + eprintln!( + "> {method} {} HTTP/1.1", + &url[url::Position::BeforePath..url::Position::AfterQuery] + ); + for (name, value) in &headers { + eprintln!("> {name}: {}", String::from_utf8_lossy(value)); + } + } + let response = send( + &url, + &method, + &headers, + if send_data { Some(&data) } else { None }, + deadline, + )?; + if args.verbose { + for line in String::from_utf8_lossy(&header_block(&response)).lines() { + eprintln!("< {line}"); + } + } + if args.include || args.head { + bytes.extend(header_block(&response)); + } + let location = response + .headers + .iter() + .find(|(n, _)| n.eq_ignore_ascii_case("location")); + if args.location + && matches!(response.status, 301 | 302 | 303 | 307 | 308) + && let Some((_, location)) = location + { + if args.max_redirs >= 0 && hops >= args.max_redirs { + return Err(error( + 47, + format!("Maximum ({}) redirects followed", args.max_redirs), + )); + } + hops += 1; + let next = url + .join(&String::from_utf8_lossy(location)) + .map_err(|_| error(3, "Invalid redirect URL"))?; + if !matches!(next.scheme(), "http" | "https") { + return Err(error(1, "Redirect protocol not supported")); + } + if next.origin() != url.origin() { + headers.retain(|(n, _)| { + !n.eq_ignore_ascii_case("authorization") && !n.eq_ignore_ascii_case("cookie") + }); + } + if response.status == 303 && method != "HEAD" + || matches!(response.status, 301 | 302) && method == "POST" + { + send_data = false; + method = args.method.clone().unwrap_or_else(|| "GET".into()); + headers.retain(|(n, _)| { + !n.eq_ignore_ascii_case("content-type") + && !n.eq_ignore_ascii_case("content-length") + }); + } + url = next; + continue; + } + break response; + }; + let failed = (args.fail || args.fail_with_body) && response.status >= 400; + if !(args.head || args.fail && failed) { + bytes.extend(&response.body); + } + if let Some(path) = output.filter(|p| p != "-") { + if !(args.fail && failed && bytes.is_empty()) { + std::fs::write(path, &bytes).map_err(|e| error(23, e.to_string()))?; + } + } else { + std::io::stdout() + .write_all(&bytes) + .map_err(|e| error(23, e.to_string()))?; + } + if let Some(format) = &args.write_out { + let content_type = response + .headers + .iter() + .find(|(n, _)| n.eq_ignore_ascii_case("content-type")) + .map(|(_, v)| String::from_utf8_lossy(v).into_owned()) + .unwrap_or_default(); + let format = format + .replace("%{http_code}", &response.status.to_string()) + .replace("%{url_effective}", url.as_str()) + .replace("%{content_type}", &content_type) + .replace( + "%{size_download}", + &if args.fail && failed { + 0 + } else { + response.body.len() + } + .to_string(), + ) + .replace("\\n", "\n"); + print!("{format}"); + } + if failed { + return Err(error( + 22, + format!("The requested URL returned error: {}", response.status), + )); + } + Ok(()) +} diff --git a/userland/curl/src/main.rs b/userland/curl/src/main.rs new file mode 100644 index 0000000..8c14b4c --- /dev/null +++ b/userland/curl/src/main.rs @@ -0,0 +1,46 @@ +//! Curl argument parsing and process exit codes. HTTP behavior lives in the library. +use clap::{CommandFactory, FromArgMatches}; +fn main() { + let args = match curl::Args::command().try_get_matches() { + Ok(matches) => { + let mut args = curl::Args::from_arg_matches(&matches).unwrap(); + let mut ordered = Vec::new(); + for (name, kind) in [("data", 0), ("raw", 1), ("binary", 2), ("json", 3)] { + if let Some(indices) = matches.indices_of(name) { + for (index, value) in indices.zip(matches.get_many::(name).unwrap()) { + ordered.push((index, value.clone(), kind)); + } + } + } + ordered.sort_by_key(|p| p.0); + args.parts = ordered + .into_iter() + .map(|(_, value, kind)| (value, kind)) + .collect(); + args + } + Err(error) => { + if !error.use_stderr() { + print!("{error}"); + return; + } + if error.kind() == clap::error::ErrorKind::UnknownArgument { + if let Some(option) = error.get(clap::error::ContextKind::InvalidArg) { + eprintln!("curl: option {option}: is unknown"); + eprintln!("curl: try 'curl --help' for more information"); + } else { + eprintln!("curl: {error}"); + } + } else { + eprintln!("curl: {error}"); + } + std::process::exit(2); + } + }; + if let Err(error) = curl::run(&args) { + if !args.silent || args.show_error { + eprintln!("curl: ({}) {}", error.code, error.message); + } + std::process::exit(error.code); + } +} diff --git a/userland/curl/test.nu b/userland/curl/test.nu new file mode 100644 index 0000000..9af2245 --- /dev/null +++ b/userland/curl/test.nu @@ -0,0 +1,93 @@ +#!/usr/bin/env nu +# Compare curl transfers with native curl and check standard WASI imports. +use std/assert +def guest [component: path, root: path, args: list, input: string = ''] { + $input | ^wasmtime run -S http -S cli-exit-with-code=y --dir $'($root)::/' --env HOME=/ $component ...$args | complete +} +def compare [name: string, component: path, root: path, url: string, args: list, input: string = '', stderr: bool = false] { + let native = ($input | ^curl ...$args $url | complete) + let wasi = (guest $component $root ($args | append $url) $input) + assert equal $wasi.exit_code $native.exit_code $name + assert equal $wasi.stdout $native.stdout $name + if $stderr { assert equal $wasi.stderr $native.stderr $name } + print $'($name): PASS' +} +def main [component: path = '/tmp/p1-curl/target/wasm32-wasip2/release/curl.wasm', readiness: path = '/tmp/p1-curl/port'] { + let component = ($component | path expand) + let root = (^mktemp -d -p /tmp/p1-curl comparison.XXXXXX | str trim) + let port = (open --raw $readiness | str trim) + let base = $'http://127.0.0.1:($port)' + let wit = (^wasm-tools component wit $component | complete) + assert equal $wit.exit_code 0 + let imports = ($wit.stdout | lines | str trim | where {|line| $line | str starts-with 'import '}) + assert (not ($imports | is-empty)) + assert ($imports | all {|line| $line | str starts-with 'import wasi:'}) + print 'Imports: PASS (WASI only)' + print ($imports | str join (char newline)) + let cases = [ + {name: GET, path: /echo, args: [-sS -A Agent]} + {name: method, path: /echo, args: [-sS -A Agent -X DELETE]} + {name: headers, path: /echo, args: [-sS -A Agent -H 'X-Test: one' -H 'X-Test: two' -H 'Accept: custom']} + {name: suppress-header, path: /echo, args: [-sS -A Agent -H 'Accept:']} + {name: data, path: /echo, args: [-sS -A Agent -d a=1 --data b=2]} + {name: empty-data, path: /echo, args: [-sS -A Agent -d '' -d x]} + {name: raw, path: /echo, args: [-sS -A Agent --data-raw '@literal']} + {name: mixed-data, path: /echo, args: [-sS -A Agent --data-raw a -d b --data-binary c]} + {name: explicit-GET-data, path: /echo, args: [-sS -A Agent -X GET -d text]} + {name: JSON, path: /echo, args: [-sS -A Agent --json '{"a":1}']} + {name: JSON-headers, path: /echo, args: [-sS -A Agent --json '{}' -H 'Content-Type: custom']} + {name: basic, path: /echo, args: [-sS -A Agent -u 'user:pass']} + {name: redirect-no-follow, path: /redirect, args: [-sS -A Agent]} + {name: redirect, path: /redirect, args: [-sS -A Agent -L]} + {name: redirect-POST-GET, path: /303, args: [-sS -A Agent -L -d x]} + {name: redirect-preserve-POST, path: /307, args: [-sS -A Agent -L -d x]} + {name: redirect-limit, path: /loop, args: [-s -L --max-redirs 1]} + {name: fail, path: /404, args: [-sS -f]} + {name: fail-body, path: /404, args: [-sS --fail-with-body]} + {name: silent-fail, path: /404, args: [-s -f]} + {name: fail-headers-size, path: /404, args: [-sS -fi -w '%{size_download}']} + {name: write-out, path: /plain, args: [-sS -w '%{http_code}|%{url_effective}|%{content_type}|%{size_download}\n']} + {name: head, path: /plain, args: [-sS -I]} + {name: include, path: /plain, args: [-sS -i]} + {name: ignored-insecure, path: /plain, args: [-sS -k]} + {name: timeout, path: /slow, args: [-s -m 0.01]} + ] + for case in $cases { compare $case.name $component $root $'($base)($case.path)' $case.args '' ($case.name in [fail fail-body silent-fail]) } + let text = $"line1\nline2\r\n" + $text | save ($root | path join input) + for flag in [-d --data-binary --json] { + let native = (^curl -sS -A Agent $flag $'@($root)/input' $'($base)/echo' | complete) + let wasi = (guest $component $root [-sS -A Agent $flag '@/input' $'($base)/echo']) + assert equal $wasi.exit_code $native.exit_code + assert equal $wasi.stdout $native.stdout + print $'($flag) @file: PASS' + compare $'($flag) @-' $component $root $'($base)/echo' [-sS -A Agent $flag '@-'] $text + } + let native = (^curl -sS -o ($root | path join native.out) $'($base)/plain' | complete) + let wasi = (guest $component $root [-sS -o /guest.out $'($base)/plain']) + assert equal $wasi.exit_code $native.exit_code + assert equal $wasi.stdout $native.stdout + assert equal (open --raw ($root | path join guest.out)) (open --raw ($root | path join native.out)) + print '-o: PASS' + let wasi = (guest $component $root [-sS -O $'($base)/plain']) + assert equal $wasi.exit_code 0 + assert equal (open --raw ($root | path join plain)) $"hello\n" + print '-O: PASS' + let native = (^curl --no-such-option | complete) + let wasi = (guest $component $root [--no-such-option]) + assert equal $wasi.exit_code 2 + assert equal $wasi.stderr $native.stderr + print 'Unknown option: PASS' + let verbose = (guest $component $root [-sS -v -H 'X-Test: visible' $'($base)/plain']) + assert equal $verbose.exit_code 0 + assert ($verbose.stderr | str contains '> X-Test: visible') + assert ($verbose.stderr | str contains '< HTTP/1.1 200 OK') + assert (not ($verbose.stderr | str contains 'Authorization')) + print 'Verbose request and response headers: PASS' + let https = (guest $component $root [-sS -w '%{http_code}' https://example.com]) + assert equal $https.exit_code 0 + assert ($https.stdout | str contains 'Example Domain') + assert ($https.stdout | str ends-with '200') + print 'Public HTTPS GET: PASS' + print $'Fixture retained: ($root)' +} diff --git a/userland/curl/tests/server/Cargo.lock b/userland/curl/tests/server/Cargo.lock new file mode 100644 index 0000000..369f900 --- /dev/null +++ b/userland/curl/tests/server/Cargo.lock @@ -0,0 +1,105 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "curl-test-server" +version = "0.1.0" +dependencies = [ + "serde_json", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.154" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7e9cc8b1b85264074fbcc02a88680c4096b1e47df8f739dceb03bf482f04bd6" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "syn" +version = "3.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d78c8dee4c7bf0e14673097256fed6142ce9d3b85a408189d07482442145823b" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "unicode-ident" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2c754d6c33795a1c324727428e5a7dedb5b06195f9890bdbcba760d3e246563" + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/userland/curl/tests/server/Cargo.toml b/userland/curl/tests/server/Cargo.toml new file mode 100644 index 0000000..eb7ca6c --- /dev/null +++ b/userland/curl/tests/server/Cargo.toml @@ -0,0 +1,7 @@ +[package] +name = "curl-test-server" +version = "0.1.0" +edition = "2024" +[workspace] +[dependencies] +serde_json = "1" diff --git a/userland/curl/tests/server/src/main.rs b/userland/curl/tests/server/src/main.rs new file mode 100644 index 0000000..d010e2a --- /dev/null +++ b/userland/curl/tests/server/src/main.rs @@ -0,0 +1,93 @@ +//! Deterministic native HTTP fixture for curl comparisons. +use std::{ + collections::BTreeMap, + io::{BufRead, BufReader, Read, Write}, + net::TcpListener, +}; +fn main() { + let ready = std::env::args().nth(1).expect("readiness path"); + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + std::fs::write(ready, listener.local_addr().unwrap().port().to_string()).unwrap(); + for stream in listener.incoming() { + let mut stream = stream.unwrap(); + let mut read = BufReader::new(stream.try_clone().unwrap()); + let mut first = String::new(); + read.read_line(&mut first).unwrap(); + let words: Vec<_> = first.split_whitespace().collect(); + if words.len() < 2 { + continue; + } + let method = words[0]; + let path = words[1]; + let mut headers: BTreeMap> = BTreeMap::new(); + let mut length = 0; + loop { + let mut line = String::new(); + read.read_line(&mut line).unwrap(); + if line == "\r\n" || line.is_empty() { + break; + } + let (name, value) = line.split_once(':').unwrap(); + let name = name.to_ascii_lowercase(); + let value = value.trim(); + if name == "content-length" { + length = value.parse().unwrap(); + } + if [ + "accept", + "content-type", + "authorization", + "user-agent", + "x-test", + ] + .contains(&name.as_str()) + { + headers.entry(name).or_default().push(value.into()); + } + } + let mut body = vec![0; length]; + read.read_exact(&mut body).unwrap(); + let echo = serde_json::json!({"method":method,"path":path,"body":String::from_utf8_lossy(&body),"headers":headers}).to_string(); + let (status, reason, response, extra) = match path { + "/plain" => (200, "OK", "hello\n".into(), "".into()), + "/404" => (404, "Not Found", "missing\n".into(), "".into()), + "/redirect" => ( + 302, + "Found", + "redirect\n".into(), + "location: /echo\r\n".into(), + ), + "/303" => ( + 303, + "See Other", + "redirect\n".into(), + "location: /echo\r\n".into(), + ), + "/307" => ( + 307, + "Temporary Redirect", + "redirect\n".into(), + "location: /echo\r\n".into(), + ), + "/loop" => ( + 302, + "Found", + "redirect\n".into(), + "location: /loop\r\n".into(), + ), + "/slow" => { + std::thread::sleep(std::time::Duration::from_millis(100)); + (200, "OK", "slow\n".into(), "".into()) + } + _ => (200, "OK", echo, "".into()), + }; + let response: String = response; + let extra: String = extra; + let _ = write!( + stream, + "HTTP/1.1 {status} {reason}\r\ncontent-type: text/plain\r\ncontent-length: {}\r\n{extra}\r\n{}", + response.len(), + if method == "HEAD" { "" } else { &response } + ); + } +} From 47948d7047619af478d140074ebf2c12b772a427 Mon Sep 17 00:00:00 2001 From: Surma Date: Fri, 6 Nov 2026 22:41:47 +0000 Subject: [PATCH 2/2] Frame curl requests and scope the test fixture lifecycle --- userland/curl/README.md | 13 ++++++++----- userland/curl/src/lib.rs | 18 +++++++++--------- userland/curl/test.nu | 24 ++++++++++++++++++++++-- 3 files changed, 39 insertions(+), 16 deletions(-) diff --git a/userland/curl/README.md b/userland/curl/README.md index e7a6f32..c1b0486 100644 --- a/userland/curl/README.md +++ b/userland/curl/README.md @@ -4,6 +4,10 @@ This CLI sends HTTP and HTTPS requests through standard WASI HTTP. The host supplies TLS and transparent credential injection. The CLI does not request or read host credentials. Each redirect creates a separate request, so the host checks every hop. +Every request has Content-Length, including empty requests. +The client finishes the body before it waits for a response. +It does not flush or poll the outgoing stream after its final byte. +A fast peer can close that stream after it consumes the framed body. Verbose request output contains only headers that the CLI supplies. ## Supported options @@ -66,18 +70,17 @@ The shared `cwd-init/init.o` must exist first. Generate it from the repository r ```sh nix develop -c nu userland/cwd-init/build.nu -CARGO_BUILD_JOBS=2 CARGO_TARGET_DIR=/tmp/p1-curl/server-target nix develop -c cargo build --manifest-path userland/curl/tests/server/Cargo.toml --locked -mkdir -p /tmp/p1-curl -/tmp/p1-curl/server-target/debug/curl-test-server /tmp/p1-curl/port ``` -Run the server as a managed job or in another terminal. -Then run the comparisons from the repository root: +Run the comparisons from the repository root: ```sh nix develop -c nu userland/curl/test.nu ``` +The script builds the native fixture with a persistent cache and starts it on a free port. +Each run uses a fresh readiness file. +The script stops the fixture after success or failure. The script compares deterministic local responses with native curl. It also checks a public HTTPS GET and verifies WASI-only component imports. The script retains its temporary files for inspection. diff --git a/userland/curl/src/lib.rs b/userland/curl/src/lib.rs index e0e0e50..d96d305 100644 --- a/userland/curl/src/lib.rs +++ b/userland/curl/src/lib.rs @@ -132,11 +132,13 @@ fn send( .append(name, value) .map_err(|_| error(2, "Invalid HTTP header"))?; } - if let Some(data) = data { - fields - .set("content-length", &[data.len().to_string().into_bytes()]) - .map_err(|_| error(2, "Invalid content length"))?; - } + // Frame even empty requests so Hyper does not discard an unknown-length body. + fields + .set( + "content-length", + &[data.map_or(0, <[u8]>::len).to_string().into_bytes()], + ) + .map_err(|_| error(2, "Invalid content length"))?; let request = OutgoingRequest::new(fields); let method = match method { "GET" => Method::Get, @@ -204,10 +206,8 @@ fn send( .map_err(|_| error(55, "Request write failed"))?; remaining = &remaining[count..]; } - stream - .flush() - .map_err(|_| error(55, "Request flush failed"))?; - wait(stream.subscribe(), deadline)?; + // After the last byte, the peer can already answer and close its reader. + // Do not flush or poll that reader after the framed body is complete. drop(stream); } OutgoingBody::finish(outgoing, None).map_err(transport_error)?; diff --git a/userland/curl/test.nu b/userland/curl/test.nu index 9af2245..ebec198 100644 --- a/userland/curl/test.nu +++ b/userland/curl/test.nu @@ -12,10 +12,30 @@ def compare [name: string, component: path, root: path, url: string, args: list, if $stderr { assert equal $wasi.stderr $native.stderr $name } print $'($name): PASS' } -def main [component: path = '/tmp/p1-curl/target/wasm32-wasip2/release/curl.wasm', readiness: path = '/tmp/p1-curl/port'] { +def main [component: path = '/tmp/p1-curl/target/wasm32-wasip2/release/curl.wasm'] { let component = ($component | path expand) + mkdir /tmp/p1-curl let root = (^mktemp -d -p /tmp/p1-curl comparison.XXXXXX | str trim) - let port = (open --raw $readiness | str trim) + let manifest = ($env.FILE_PWD | path join tests/server/Cargo.toml) + let build = (with-env {CARGO_BUILD_JOBS: '2', CARGO_TARGET_DIR: '/tmp/p1-curl/server-target'} { ^cargo build --manifest-path $manifest --locked } | complete) + if $build.exit_code != 0 { error make {msg: ($build | to json)} } + let readiness = ($root | path join port) + let server = job spawn { ^/tmp/p1-curl/server-target/debug/curl-test-server $readiness out> ($root | path join server.out) err> ($root | path join server.err) } + try { + let deadline = (date now) + 5sec + while not ($readiness | path exists) { + if (date now) >= $deadline { error make {msg: $'Fixture did not start. Inspect ($root)/server.err'} } + } + let port = (open --raw $readiness | str trim) + run-tests $component $root $port + } catch {|failure| + job kill $server + error make {msg: $failure.msg} + } + job kill $server + print 'Fixture stopped: PASS' +} +def run-tests [component: path, root: path, port: string] { let base = $'http://127.0.0.1:($port)' let wit = (^wasm-tools component wit $component | complete) assert equal $wit.exit_code 0