diff --git a/userland/Cargo.lock b/userland/Cargo.lock index c463566..8ac2d5d 100644 --- a/userland/Cargo.lock +++ b/userland/Cargo.lock @@ -744,6 +744,16 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "curl" +version = "0.1.0" +dependencies = [ + "base64", + "clap", + "url", + "wasi 0.14.7+wasi-0.2.4", +] + [[package]] name = "curve25519-dalek" version = "4.1.3" diff --git a/userland/Cargo.toml b/userland/Cargo.toml index f20996a..10f72ab 100644 --- a/userland/Cargo.toml +++ b/userland/Cargo.toml @@ -1,7 +1,7 @@ # The default CLIs of a world, built for wasm32-wasip2. Its own workspace, outside the host's. [workspace] resolver = "3" -members = ["git", "coreutils", "procps", "cwd-init/probe", "ripgrep", "diffutils", "jaq", "sed", "grep", "fd", "spawn", "findutils", "exec-tools", "ssh-client", "ssh"] +members = ["git", "coreutils", "procps", "cwd-init/probe", "ripgrep", "diffutils", "jaq", "sed", "grep", "fd", "spawn", "findutils", "exec-tools", "ssh-client", "ssh", "curl"] # WASI compatibility patches: git/vendor/PATCHES.md. [patch.crates-io] diff --git a/userland/curl/Cargo.toml b/userland/curl/Cargo.toml new file mode 100644 index 0000000..6999e9b --- /dev/null +++ b/userland/curl/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "curl" +version = "0.1.0" +edition = "2024" + +[dependencies] +clap = { version = "4", features = ["derive"] } +url = "2" +wasi = "0.14" +base64 = "0.22" diff --git a/userland/curl/README.md b/userland/curl/README.md new file mode 100644 index 0000000..c1b0486 --- /dev/null +++ b/userland/curl/README.md @@ -0,0 +1,86 @@ +# Curl for the world + +This CLI sends HTTP and HTTPS requests through standard WASI HTTP. +The host supplies TLS and transparent credential injection. +The CLI does not request or read host credentials. +Each redirect creates a separate request, so the host checks every hop. +Every request has Content-Length, including empty requests. +The client finishes the body before it waits for a response. +It does not flush or poll the outgoing stream after its final byte. +A fast peer can close that stream after it consumes the framed body. +Verbose request output contains only headers that the CLI supplies. + +## Supported options + +- A positional HTTP or HTTPS URL. A URL without a scheme defaults to HTTP. +- `-X`, `--request` selects the method. +- Repeatable `-H`, `--header` overrides defaults or adds headers. An empty value suppresses a default header. +- `-d`, `--data` reads literal data, `@file`, or `@-`. File input removes CR, LF, and NUL bytes. +- `--data-raw` treats all input literally, including an initial `@`. +- `--data-binary` preserves file and stdin bytes. +- Data options imply POST and the form content type. Repeated data options join with `&` in argument order. +- `--json` implies POST, preserves its body, and sets JSON Content-Type and Accept headers. +- `-o`, `--output` writes to a world file. `-o -` writes to stdout. +- `-O`, `--remote-name` uses the original URL's last path segment. +- `-s`, `--silent` suppresses errors. `-S`, `--show-error` restores errors with silent mode. +- `-i`, `--include` includes response headers. `-I`, `--head` selects HEAD and prints headers. +- `-L`, `--location` follows redirects. `--max-redirs` defaults to 50. A negative value removes the limit. +- Redirects 301, 302, and 303 convert POST to GET. Redirects 307 and 308 preserve the body. +- Redirects to another origin remove explicit Authorization and Cookie headers. +- `-f`, `--fail` suppresses error bodies and returns 22 for status codes of at least 400. +- `--fail-with-body` returns 22 but retains the body. +- `-w`, `--write-out` supports `%{http_code}`, `%{url_effective}`, `%{content_type}`, `%{size_download}`, and `\n`. +- `-v`, `--verbose` prints request and response headers to stderr. +- `-u`, `--user` supplies Basic authentication. A missing password uses an empty password without a prompt. +- `-A`, `--user-agent` supplies the User-Agent header. +- `-k`, `--insecure` is accepted and ignored. The host controls TLS verification. +- `-m`, `--max-time` bounds network operations across redirects. Zero removes the deadline. +- `--help` prints the CLI's supported options. + +Unknown options return 2 with curl's unknown-option message. +HTTP failures return 22 only when a fail option applies. +Network deadlines return 28. Input errors return 26. Output errors return 23. + +## Known differences + +The CLI supports one URL per invocation. +It does not implement multipart `-F`, `--compressed`, or curl's other options. +It buffers request and response bodies. It does not show a progress meter. +The default User-Agent is `curl/8.0.0`, not the native curl version. +WASI exposes the status code, but not the HTTP version or reason phrase. +The CLI reconstructs header status lines as HTTP/1.1 with standard reason phrases for common codes. +The host can normalize response header names and order. +WASI omits hop-by-hop headers, such as Connection, from the response. +WASI rejects forbidden request headers. The CLI sets Content-Length from the body size. +Verbose output omits native curl's connection and TLS diagnostics. +Transport diagnostics describe WASI errors instead of native libcurl errors. +Argument diagnostics other than unknown options use clap's text with exit code 2. +Write-out supports only the listed substitutions and a literal format string. +The CLI does not implement curl's credential prompts, cookie jar, config file, or proxy options. + +## Validation + +Run these commands from `userland/`: + +```sh +CARGO_BUILD_JOBS=2 CARGO_TARGET_DIR=/tmp/p1-curl/target nix develop .. -c cargo build -p curl --release --locked +``` + +The shared `cwd-init/init.o` must exist first. Generate it from the repository root if needed: + +```sh +nix develop -c nu userland/cwd-init/build.nu +``` + +Run the comparisons from the repository root: + +```sh +nix develop -c nu userland/curl/test.nu +``` + +The script builds the native fixture with a persistent cache and starts it on a free port. +Each run uses a fresh readiness file. +The script stops the fixture after success or failure. +The script compares deterministic local responses with native curl. +It also checks a public HTTPS GET and verifies WASI-only component imports. +The script retains its temporary files for inspection. diff --git a/userland/curl/src/lib.rs b/userland/curl/src/lib.rs new file mode 100644 index 0000000..d96d305 --- /dev/null +++ b/userland/curl/src/lib.rs @@ -0,0 +1,516 @@ +//! HTTP transfers through WASI, without access to host-injected credentials. +use base64::{Engine, engine::general_purpose::STANDARD}; +use clap::Parser; +use std::{ + io::{Read, Write}, + time::{Duration, Instant}, +}; +use wasi::http::{outgoing_handler, types::*}; + +#[derive(Parser, Debug)] +#[command(disable_version_flag = true)] +pub struct Args { + pub url: String, + #[arg(skip)] + pub parts: Vec<(String, u8)>, + #[arg(short = 'X', long = "request")] + pub method: Option, + #[arg(short = 'H', long = "header", allow_hyphen_values = true)] + pub headers: Vec, + #[arg(short = 'd', long = "data", allow_hyphen_values = true)] + pub data: Vec, + #[arg(long = "data-raw", allow_hyphen_values = true)] + pub raw: Vec, + #[arg(long = "data-binary", allow_hyphen_values = true)] + pub binary: Vec, + #[arg(long, allow_hyphen_values = true)] + pub json: Vec, + #[arg(short = 'o', long = "output")] + pub output: Option, + #[arg(short = 'O', long = "remote-name")] + pub remote_name: bool, + #[arg(short = 's', long = "silent")] + pub silent: bool, + #[arg(short = 'S', long = "show-error")] + pub show_error: bool, + #[arg(short = 'i', long = "include")] + pub include: bool, + #[arg(short = 'I', long = "head")] + pub head: bool, + #[arg(short = 'L', long = "location")] + pub location: bool, + #[arg(long, default_value_t = 50, allow_hyphen_values = true)] + pub max_redirs: i64, + #[arg(short = 'f', long = "fail", conflicts_with = "fail_with_body")] + pub fail: bool, + #[arg(long)] + pub fail_with_body: bool, + #[arg(short = 'w', long = "write-out")] + pub write_out: Option, + #[arg(short = 'v', long = "verbose")] + pub verbose: bool, + #[arg(short = 'u', long = "user")] + pub user: Option, + #[arg(short = 'A', long = "user-agent")] + pub agent: Option, + #[arg(short = 'k', long = "insecure")] + pub insecure: bool, + #[arg(short = 'm', long = "max-time")] + pub max_time: Option, +} +pub struct Error { + pub code: i32, + pub message: String, +} +type Result = std::result::Result; +fn error(code: i32, message: impl Into) -> Error { + Error { + code, + message: message.into(), + } +} +fn file_data(text: &str, files: bool, strip: bool) -> Result> { + let mut bytes = if files && text.starts_with('@') { + if text == "@-" { + let mut bytes = Vec::new(); + std::io::stdin() + .read_to_end(&mut bytes) + .map_err(|e| error(26, e.to_string()))?; + bytes + } else { + std::fs::read(&text[1..]).map_err(|e| error(26, e.to_string()))? + } + } else { + return Ok(text.as_bytes().to_vec()); + }; + if strip { + bytes.retain(|b| !matches!(b, b'\r' | b'\n' | 0)); + } + Ok(bytes) +} +fn wait(pollable: wasi::io::poll::Pollable, deadline: Option) -> Result<()> { + if let Some(deadline) = deadline { + let remaining = deadline + .checked_duration_since(Instant::now()) + .ok_or_else(|| error(28, "Operation timed out"))?; + let timer = wasi::clocks::monotonic_clock::subscribe_duration( + remaining.as_nanos().min(u64::MAX as u128) as u64, + ); + wasi::io::poll::poll(&[&pollable, &timer]); + if Instant::now() >= deadline { + return Err(error(28, "Operation timed out")); + } + } else { + pollable.block(); + } + Ok(()) +} +fn transport_error(e: ErrorCode) -> Error { + let code = match e { + ErrorCode::DnsError(_) | ErrorCode::DestinationNotFound => 6, + ErrorCode::ConnectionTimeout | ErrorCode::HttpResponseTimeout => 28, + ErrorCode::ConnectionRefused | ErrorCode::ConnectionTerminated => 7, + _ => 56, + }; + error(code, format!("HTTP transfer failed: {e:?}")) +} +struct Response { + status: u16, + headers: Vec<(String, Vec)>, + body: Vec, +} +fn send( + url: &url::Url, + method: &str, + headers: &[(String, Vec)], + data: Option<&[u8]>, + deadline: Option, +) -> Result { + let fields = Fields::new(); + for (name, value) in headers { + fields + .append(name, value) + .map_err(|_| error(2, "Invalid HTTP header"))?; + } + // Frame even empty requests so Hyper does not discard an unknown-length body. + fields + .set( + "content-length", + &[data.map_or(0, <[u8]>::len).to_string().into_bytes()], + ) + .map_err(|_| error(2, "Invalid content length"))?; + let request = OutgoingRequest::new(fields); + let method = match method { + "GET" => Method::Get, + "POST" => Method::Post, + "HEAD" => Method::Head, + "PUT" => Method::Put, + "DELETE" => Method::Delete, + "PATCH" => Method::Patch, + "OPTIONS" => Method::Options, + other => Method::Other(other.into()), + }; + request + .set_method(&method) + .map_err(|_| error(2, "Invalid HTTP method"))?; + request + .set_scheme(Some(&if url.scheme() == "https" { + Scheme::Https + } else { + Scheme::Http + })) + .map_err(|_| error(3, "Invalid scheme"))?; + request + .set_authority(Some( + &url[url::Position::BeforeHost..url::Position::AfterPort], + )) + .map_err(|_| error(3, "Invalid authority"))?; + request + .set_path_with_query(Some( + &url[url::Position::BeforePath..url::Position::AfterQuery], + )) + .map_err(|_| error(3, "Invalid path"))?; + let outgoing = request + .body() + .map_err(|_| error(26, "Cannot open request body"))?; + let options = RequestOptions::new(); + if let Some(deadline) = deadline { + let ns = deadline + .saturating_duration_since(Instant::now()) + .as_nanos() + .min(u64::MAX as u128) as u64; + options + .set_connect_timeout(Some(ns)) + .map_err(|_| error(28, "Timeout unsupported"))?; + options + .set_first_byte_timeout(Some(ns)) + .map_err(|_| error(28, "Timeout unsupported"))?; + options + .set_between_bytes_timeout(Some(ns)) + .map_err(|_| error(28, "Timeout unsupported"))?; + } + let future = outgoing_handler::handle(request, Some(options)).map_err(transport_error)?; + if let Some(data) = data { + let stream = outgoing + .write() + .map_err(|_| error(26, "Cannot write request"))?; + let mut remaining = data; + while !remaining.is_empty() { + wait(stream.subscribe(), deadline)?; + let count = stream + .check_write() + .map_err(|_| error(55, "Request write failed"))? as usize; + let count = count.min(remaining.len()); + stream + .write(&remaining[..count]) + .map_err(|_| error(55, "Request write failed"))?; + remaining = &remaining[count..]; + } + // After the last byte, the peer can already answer and close its reader. + // Do not flush or poll that reader after the framed body is complete. + drop(stream); + } + OutgoingBody::finish(outgoing, None).map_err(transport_error)?; + wait(future.subscribe(), deadline)?; + let response = future + .get() + .ok_or_else(|| error(56, "Missing response"))? + .map_err(|_| error(56, "Consumed response"))? + .map_err(transport_error)?; + let status = response.status(); + let headers = response.headers().entries(); + let incoming = response + .consume() + .map_err(|_| error(56, "Cannot consume response"))?; + let stream = incoming + .stream() + .map_err(|_| error(56, "Cannot read response"))?; + let mut body = Vec::new(); + loop { + wait(stream.subscribe(), deadline)?; + match stream.read(65536) { + Ok(bytes) => body.extend(bytes), + Err(wasi::io::streams::StreamError::Closed) => break, + Err(_) => return Err(error(56, "Response read failed")), + } + } + drop(stream); + let trailers = IncomingBody::finish(incoming); + wait(trailers.subscribe(), deadline)?; + trailers + .get() + .ok_or_else(|| error(56, "Missing trailers"))? + .map_err(|_| error(56, "Consumed trailers"))? + .map_err(transport_error)?; + Ok(Response { + status, + headers, + body, + }) +} +fn header_block(response: &Response) -> Vec { + let reason = match response.status { + 200 => "OK", + 201 => "Created", + 204 => "No Content", + 301 => "Moved Permanently", + 302 => "Found", + 303 => "See Other", + 307 => "Temporary Redirect", + 308 => "Permanent Redirect", + 400 => "Bad Request", + 401 => "Unauthorized", + 403 => "Forbidden", + 404 => "Not Found", + 500 => "Internal Server Error", + _ => "", + }; + let mut bytes = format!("HTTP/1.1 {} {reason}\r\n", response.status).into_bytes(); + for (name, value) in &response.headers { + bytes.extend(name.as_bytes()); + bytes.extend(b": "); + bytes.extend(value); + bytes.extend(b"\r\n"); + } + bytes.extend(b"\r\n"); + bytes +} +pub fn run(args: &Args) -> Result<()> { + let mut url = url::Url::parse(&args.url) + .or_else(|_| url::Url::parse(&format!("http://{}", args.url))) + .map_err(|_| error(3, "URL rejected"))?; + if !matches!(url.scheme(), "http" | "https") { + return Err(error(1, "Protocol not supported")); + } + let deadline = match args.max_time { + Some(n) if !n.is_finite() || n < 0.0 => return Err(error(2, "Invalid max-time")), + Some(n) if n > 0.0 => Some( + Instant::now() + .checked_add( + Duration::try_from_secs_f64(n).map_err(|_| error(2, "Invalid max-time"))?, + ) + .ok_or_else(|| error(2, "Invalid max-time"))?, + ), + _ => None, + }; + let mut data = Vec::new(); + let has_data = !args.data.is_empty() + || !args.raw.is_empty() + || !args.binary.is_empty() + || !args.json.is_empty(); + for (value, kind) in &args.parts { + if !data.is_empty() && args.json.is_empty() { + data.push(b'&'); + } + data.extend(file_data(value, *kind != 1, *kind == 0)?); + } + let mut send_data = has_data; + let mut method = if args.head { + "HEAD" + } else if has_data { + "POST" + } else { + "GET" + } + .to_string(); + if let Some(custom) = &args.method { + method.clone_from(custom); + } + let mut headers: Vec<(String, Vec)> = vec![ + ( + "User-Agent".into(), + args.agent + .as_deref() + .unwrap_or("curl/8.0.0") + .as_bytes() + .to_vec(), + ), + ( + "Accept".into(), + if args.json.is_empty() { + b"*/*".to_vec() + } else { + b"application/json".to_vec() + }, + ), + ]; + if has_data { + headers.push(( + "Content-Type".into(), + if args.json.is_empty() { + b"application/x-www-form-urlencoded".to_vec() + } else { + b"application/json".to_vec() + }, + )); + } + let user = args.user.clone().or_else(|| { + if url.username().is_empty() { + None + } else { + Some(format!( + "{}:{}", + url.username(), + url.password().unwrap_or("") + )) + } + }); + if let Some(user) = user { + let user = if user.contains(':') { + user + } else { + format!("{user}:") + }; + headers.push(( + "Authorization".into(), + format!("Basic {}", STANDARD.encode(user)).into_bytes(), + )); + } + url.set_username("").map_err(|_| error(3, "Invalid URL"))?; + url.set_password(None) + .map_err(|_| error(3, "Invalid URL"))?; + let mut custom_headers = std::collections::HashSet::new(); + for h in &args.headers { + let (name, value) = h + .split_once(':') + .or_else(|| h.strip_suffix(';').map(|name| (name, ";"))) + .ok_or_else(|| error(2, "Invalid HTTP header"))?; + if custom_headers.insert(name.to_ascii_lowercase()) { + headers.retain(|(n, _)| !n.eq_ignore_ascii_case(name)); + } + if !value.is_empty() { + headers.push(( + name.into(), + if value == ";" { + Vec::new() + } else { + value.trim_start().as_bytes().to_vec() + }, + )); + } + } + let output = args.output.clone().or_else(|| { + if args.remote_name { + url.path_segments() + .and_then(|mut s| s.next_back()) + .filter(|s| !s.is_empty()) + .map(str::to_owned) + } else { + None + } + }); + if args.remote_name && output.is_none() { + return Err(error(23, "Remote file name has no length")); + } + let mut bytes = Vec::new(); + let mut hops = 0; + let response = loop { + if args.verbose { + eprintln!( + "> {method} {} HTTP/1.1", + &url[url::Position::BeforePath..url::Position::AfterQuery] + ); + for (name, value) in &headers { + eprintln!("> {name}: {}", String::from_utf8_lossy(value)); + } + } + let response = send( + &url, + &method, + &headers, + if send_data { Some(&data) } else { None }, + deadline, + )?; + if args.verbose { + for line in String::from_utf8_lossy(&header_block(&response)).lines() { + eprintln!("< {line}"); + } + } + if args.include || args.head { + bytes.extend(header_block(&response)); + } + let location = response + .headers + .iter() + .find(|(n, _)| n.eq_ignore_ascii_case("location")); + if args.location + && matches!(response.status, 301 | 302 | 303 | 307 | 308) + && let Some((_, location)) = location + { + if args.max_redirs >= 0 && hops >= args.max_redirs { + return Err(error( + 47, + format!("Maximum ({}) redirects followed", args.max_redirs), + )); + } + hops += 1; + let next = url + .join(&String::from_utf8_lossy(location)) + .map_err(|_| error(3, "Invalid redirect URL"))?; + if !matches!(next.scheme(), "http" | "https") { + return Err(error(1, "Redirect protocol not supported")); + } + if next.origin() != url.origin() { + headers.retain(|(n, _)| { + !n.eq_ignore_ascii_case("authorization") && !n.eq_ignore_ascii_case("cookie") + }); + } + if response.status == 303 && method != "HEAD" + || matches!(response.status, 301 | 302) && method == "POST" + { + send_data = false; + method = args.method.clone().unwrap_or_else(|| "GET".into()); + headers.retain(|(n, _)| { + !n.eq_ignore_ascii_case("content-type") + && !n.eq_ignore_ascii_case("content-length") + }); + } + url = next; + continue; + } + break response; + }; + let failed = (args.fail || args.fail_with_body) && response.status >= 400; + if !(args.head || args.fail && failed) { + bytes.extend(&response.body); + } + if let Some(path) = output.filter(|p| p != "-") { + if !(args.fail && failed && bytes.is_empty()) { + std::fs::write(path, &bytes).map_err(|e| error(23, e.to_string()))?; + } + } else { + std::io::stdout() + .write_all(&bytes) + .map_err(|e| error(23, e.to_string()))?; + } + if let Some(format) = &args.write_out { + let content_type = response + .headers + .iter() + .find(|(n, _)| n.eq_ignore_ascii_case("content-type")) + .map(|(_, v)| String::from_utf8_lossy(v).into_owned()) + .unwrap_or_default(); + let format = format + .replace("%{http_code}", &response.status.to_string()) + .replace("%{url_effective}", url.as_str()) + .replace("%{content_type}", &content_type) + .replace( + "%{size_download}", + &if args.fail && failed { + 0 + } else { + response.body.len() + } + .to_string(), + ) + .replace("\\n", "\n"); + print!("{format}"); + } + if failed { + return Err(error( + 22, + format!("The requested URL returned error: {}", response.status), + )); + } + Ok(()) +} diff --git a/userland/curl/src/main.rs b/userland/curl/src/main.rs new file mode 100644 index 0000000..8c14b4c --- /dev/null +++ b/userland/curl/src/main.rs @@ -0,0 +1,46 @@ +//! Curl argument parsing and process exit codes. HTTP behavior lives in the library. +use clap::{CommandFactory, FromArgMatches}; +fn main() { + let args = match curl::Args::command().try_get_matches() { + Ok(matches) => { + let mut args = curl::Args::from_arg_matches(&matches).unwrap(); + let mut ordered = Vec::new(); + for (name, kind) in [("data", 0), ("raw", 1), ("binary", 2), ("json", 3)] { + if let Some(indices) = matches.indices_of(name) { + for (index, value) in indices.zip(matches.get_many::(name).unwrap()) { + ordered.push((index, value.clone(), kind)); + } + } + } + ordered.sort_by_key(|p| p.0); + args.parts = ordered + .into_iter() + .map(|(_, value, kind)| (value, kind)) + .collect(); + args + } + Err(error) => { + if !error.use_stderr() { + print!("{error}"); + return; + } + if error.kind() == clap::error::ErrorKind::UnknownArgument { + if let Some(option) = error.get(clap::error::ContextKind::InvalidArg) { + eprintln!("curl: option {option}: is unknown"); + eprintln!("curl: try 'curl --help' for more information"); + } else { + eprintln!("curl: {error}"); + } + } else { + eprintln!("curl: {error}"); + } + std::process::exit(2); + } + }; + if let Err(error) = curl::run(&args) { + if !args.silent || args.show_error { + eprintln!("curl: ({}) {}", error.code, error.message); + } + std::process::exit(error.code); + } +} diff --git a/userland/curl/test.nu b/userland/curl/test.nu new file mode 100644 index 0000000..ebec198 --- /dev/null +++ b/userland/curl/test.nu @@ -0,0 +1,113 @@ +#!/usr/bin/env nu +# Compare curl transfers with native curl and check standard WASI imports. +use std/assert +def guest [component: path, root: path, args: list, input: string = ''] { + $input | ^wasmtime run -S http -S cli-exit-with-code=y --dir $'($root)::/' --env HOME=/ $component ...$args | complete +} +def compare [name: string, component: path, root: path, url: string, args: list, input: string = '', stderr: bool = false] { + let native = ($input | ^curl ...$args $url | complete) + let wasi = (guest $component $root ($args | append $url) $input) + assert equal $wasi.exit_code $native.exit_code $name + assert equal $wasi.stdout $native.stdout $name + if $stderr { assert equal $wasi.stderr $native.stderr $name } + print $'($name): PASS' +} +def main [component: path = '/tmp/p1-curl/target/wasm32-wasip2/release/curl.wasm'] { + let component = ($component | path expand) + mkdir /tmp/p1-curl + let root = (^mktemp -d -p /tmp/p1-curl comparison.XXXXXX | str trim) + let manifest = ($env.FILE_PWD | path join tests/server/Cargo.toml) + let build = (with-env {CARGO_BUILD_JOBS: '2', CARGO_TARGET_DIR: '/tmp/p1-curl/server-target'} { ^cargo build --manifest-path $manifest --locked } | complete) + if $build.exit_code != 0 { error make {msg: ($build | to json)} } + let readiness = ($root | path join port) + let server = job spawn { ^/tmp/p1-curl/server-target/debug/curl-test-server $readiness out> ($root | path join server.out) err> ($root | path join server.err) } + try { + let deadline = (date now) + 5sec + while not ($readiness | path exists) { + if (date now) >= $deadline { error make {msg: $'Fixture did not start. Inspect ($root)/server.err'} } + } + let port = (open --raw $readiness | str trim) + run-tests $component $root $port + } catch {|failure| + job kill $server + error make {msg: $failure.msg} + } + job kill $server + print 'Fixture stopped: PASS' +} +def run-tests [component: path, root: path, port: string] { + let base = $'http://127.0.0.1:($port)' + let wit = (^wasm-tools component wit $component | complete) + assert equal $wit.exit_code 0 + let imports = ($wit.stdout | lines | str trim | where {|line| $line | str starts-with 'import '}) + assert (not ($imports | is-empty)) + assert ($imports | all {|line| $line | str starts-with 'import wasi:'}) + print 'Imports: PASS (WASI only)' + print ($imports | str join (char newline)) + let cases = [ + {name: GET, path: /echo, args: [-sS -A Agent]} + {name: method, path: /echo, args: [-sS -A Agent -X DELETE]} + {name: headers, path: /echo, args: [-sS -A Agent -H 'X-Test: one' -H 'X-Test: two' -H 'Accept: custom']} + {name: suppress-header, path: /echo, args: [-sS -A Agent -H 'Accept:']} + {name: data, path: /echo, args: [-sS -A Agent -d a=1 --data b=2]} + {name: empty-data, path: /echo, args: [-sS -A Agent -d '' -d x]} + {name: raw, path: /echo, args: [-sS -A Agent --data-raw '@literal']} + {name: mixed-data, path: /echo, args: [-sS -A Agent --data-raw a -d b --data-binary c]} + {name: explicit-GET-data, path: /echo, args: [-sS -A Agent -X GET -d text]} + {name: JSON, path: /echo, args: [-sS -A Agent --json '{"a":1}']} + {name: JSON-headers, path: /echo, args: [-sS -A Agent --json '{}' -H 'Content-Type: custom']} + {name: basic, path: /echo, args: [-sS -A Agent -u 'user:pass']} + {name: redirect-no-follow, path: /redirect, args: [-sS -A Agent]} + {name: redirect, path: /redirect, args: [-sS -A Agent -L]} + {name: redirect-POST-GET, path: /303, args: [-sS -A Agent -L -d x]} + {name: redirect-preserve-POST, path: /307, args: [-sS -A Agent -L -d x]} + {name: redirect-limit, path: /loop, args: [-s -L --max-redirs 1]} + {name: fail, path: /404, args: [-sS -f]} + {name: fail-body, path: /404, args: [-sS --fail-with-body]} + {name: silent-fail, path: /404, args: [-s -f]} + {name: fail-headers-size, path: /404, args: [-sS -fi -w '%{size_download}']} + {name: write-out, path: /plain, args: [-sS -w '%{http_code}|%{url_effective}|%{content_type}|%{size_download}\n']} + {name: head, path: /plain, args: [-sS -I]} + {name: include, path: /plain, args: [-sS -i]} + {name: ignored-insecure, path: /plain, args: [-sS -k]} + {name: timeout, path: /slow, args: [-s -m 0.01]} + ] + for case in $cases { compare $case.name $component $root $'($base)($case.path)' $case.args '' ($case.name in [fail fail-body silent-fail]) } + let text = $"line1\nline2\r\n" + $text | save ($root | path join input) + for flag in [-d --data-binary --json] { + let native = (^curl -sS -A Agent $flag $'@($root)/input' $'($base)/echo' | complete) + let wasi = (guest $component $root [-sS -A Agent $flag '@/input' $'($base)/echo']) + assert equal $wasi.exit_code $native.exit_code + assert equal $wasi.stdout $native.stdout + print $'($flag) @file: PASS' + compare $'($flag) @-' $component $root $'($base)/echo' [-sS -A Agent $flag '@-'] $text + } + let native = (^curl -sS -o ($root | path join native.out) $'($base)/plain' | complete) + let wasi = (guest $component $root [-sS -o /guest.out $'($base)/plain']) + assert equal $wasi.exit_code $native.exit_code + assert equal $wasi.stdout $native.stdout + assert equal (open --raw ($root | path join guest.out)) (open --raw ($root | path join native.out)) + print '-o: PASS' + let wasi = (guest $component $root [-sS -O $'($base)/plain']) + assert equal $wasi.exit_code 0 + assert equal (open --raw ($root | path join plain)) $"hello\n" + print '-O: PASS' + let native = (^curl --no-such-option | complete) + let wasi = (guest $component $root [--no-such-option]) + assert equal $wasi.exit_code 2 + assert equal $wasi.stderr $native.stderr + print 'Unknown option: PASS' + let verbose = (guest $component $root [-sS -v -H 'X-Test: visible' $'($base)/plain']) + assert equal $verbose.exit_code 0 + assert ($verbose.stderr | str contains '> X-Test: visible') + assert ($verbose.stderr | str contains '< HTTP/1.1 200 OK') + assert (not ($verbose.stderr | str contains 'Authorization')) + print 'Verbose request and response headers: PASS' + let https = (guest $component $root [-sS -w '%{http_code}' https://example.com]) + assert equal $https.exit_code 0 + assert ($https.stdout | str contains 'Example Domain') + assert ($https.stdout | str ends-with '200') + print 'Public HTTPS GET: PASS' + print $'Fixture retained: ($root)' +} diff --git a/userland/curl/tests/server/Cargo.lock b/userland/curl/tests/server/Cargo.lock new file mode 100644 index 0000000..369f900 --- /dev/null +++ b/userland/curl/tests/server/Cargo.lock @@ -0,0 +1,105 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "curl-test-server" +version = "0.1.0" +dependencies = [ + "serde_json", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.154" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7e9cc8b1b85264074fbcc02a88680c4096b1e47df8f739dceb03bf482f04bd6" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "syn" +version = "3.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d78c8dee4c7bf0e14673097256fed6142ce9d3b85a408189d07482442145823b" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "unicode-ident" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2c754d6c33795a1c324727428e5a7dedb5b06195f9890bdbcba760d3e246563" + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/userland/curl/tests/server/Cargo.toml b/userland/curl/tests/server/Cargo.toml new file mode 100644 index 0000000..eb7ca6c --- /dev/null +++ b/userland/curl/tests/server/Cargo.toml @@ -0,0 +1,7 @@ +[package] +name = "curl-test-server" +version = "0.1.0" +edition = "2024" +[workspace] +[dependencies] +serde_json = "1" diff --git a/userland/curl/tests/server/src/main.rs b/userland/curl/tests/server/src/main.rs new file mode 100644 index 0000000..d010e2a --- /dev/null +++ b/userland/curl/tests/server/src/main.rs @@ -0,0 +1,93 @@ +//! Deterministic native HTTP fixture for curl comparisons. +use std::{ + collections::BTreeMap, + io::{BufRead, BufReader, Read, Write}, + net::TcpListener, +}; +fn main() { + let ready = std::env::args().nth(1).expect("readiness path"); + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + std::fs::write(ready, listener.local_addr().unwrap().port().to_string()).unwrap(); + for stream in listener.incoming() { + let mut stream = stream.unwrap(); + let mut read = BufReader::new(stream.try_clone().unwrap()); + let mut first = String::new(); + read.read_line(&mut first).unwrap(); + let words: Vec<_> = first.split_whitespace().collect(); + if words.len() < 2 { + continue; + } + let method = words[0]; + let path = words[1]; + let mut headers: BTreeMap> = BTreeMap::new(); + let mut length = 0; + loop { + let mut line = String::new(); + read.read_line(&mut line).unwrap(); + if line == "\r\n" || line.is_empty() { + break; + } + let (name, value) = line.split_once(':').unwrap(); + let name = name.to_ascii_lowercase(); + let value = value.trim(); + if name == "content-length" { + length = value.parse().unwrap(); + } + if [ + "accept", + "content-type", + "authorization", + "user-agent", + "x-test", + ] + .contains(&name.as_str()) + { + headers.entry(name).or_default().push(value.into()); + } + } + let mut body = vec![0; length]; + read.read_exact(&mut body).unwrap(); + let echo = serde_json::json!({"method":method,"path":path,"body":String::from_utf8_lossy(&body),"headers":headers}).to_string(); + let (status, reason, response, extra) = match path { + "/plain" => (200, "OK", "hello\n".into(), "".into()), + "/404" => (404, "Not Found", "missing\n".into(), "".into()), + "/redirect" => ( + 302, + "Found", + "redirect\n".into(), + "location: /echo\r\n".into(), + ), + "/303" => ( + 303, + "See Other", + "redirect\n".into(), + "location: /echo\r\n".into(), + ), + "/307" => ( + 307, + "Temporary Redirect", + "redirect\n".into(), + "location: /echo\r\n".into(), + ), + "/loop" => ( + 302, + "Found", + "redirect\n".into(), + "location: /loop\r\n".into(), + ), + "/slow" => { + std::thread::sleep(std::time::Duration::from_millis(100)); + (200, "OK", "slow\n".into(), "".into()) + } + _ => (200, "OK", echo, "".into()), + }; + let response: String = response; + let extra: String = extra; + let _ = write!( + stream, + "HTTP/1.1 {status} {reason}\r\ncontent-type: text/plain\r\ncontent-length: {}\r\n{extra}\r\n{}", + response.len(), + if method == "HEAD" { "" } else { &response } + ); + } +}