diff --git a/userland/Cargo.lock b/userland/Cargo.lock index 0ffc254..6840a5c 100644 --- a/userland/Cargo.lock +++ b/userland/Cargo.lock @@ -38,7 +38,7 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "90759a65f5178dbbd1b0eecc97b187996401bced0eb4a943d9ac6d7a6304d111" dependencies = [ - "unicode-width", + "unicode-width 0.2.3", ] [[package]] @@ -77,7 +77,7 @@ version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" dependencies = [ - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -88,7 +88,7 @@ checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" dependencies = [ "anstyle", "once_cell_polyfill", - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -97,6 +97,15 @@ version = "1.0.104" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" +[[package]] +name = "approx" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cab112f0a86d568ea0e627cc1d6be74a1e9cd55214684db5561995f6dad897c6" +dependencies = [ + "num-traits", +] + [[package]] name = "arc-swap" version = "1.9.3" @@ -106,12 +115,48 @@ dependencies = [ "rustversion", ] +[[package]] +name = "argmax" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0144c58b55af0133ec3963ce5e4d07aad866e3bbcfdcddbf4590dbd7ad6ff557" +dependencies = [ + "libc", + "nix 0.30.1", + "once_cell", +] + +[[package]] +name = "ariadne" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8454c8a44ce2cb9cc7e7fae67fc6128465b343b92c6631e94beca3c8d1524ea5" +dependencies = [ + "unicode-width 0.2.3", + "yansi", +] + [[package]] name = "arrayvec" version = "0.7.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56" +[[package]] +name = "assert_cmd" +version = "2.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2aa3a22042e45de04255c7bf3626e239f450200fd0493c1e382263544b20aea6" +dependencies = [ + "anstyle", + "bstr", + "libc", + "predicates", + "predicates-core", + "predicates-tree", + "wait-timeout", +] + [[package]] name = "autocfg" version = "1.5.1" @@ -279,7 +324,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4ff2a8126fff0f8c839080d619675a585e61720bb12d2baceae45769fd66f155" dependencies = [ "find-msvc-tools", - "shlex", + "jobserver", + "libc", + "shlex 2.0.1", ] [[package]] @@ -312,7 +359,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" dependencies = [ "iana-time-zone", + "js-sys", "num-traits", + "wasm-bindgen", "windows-link", ] @@ -339,6 +388,15 @@ dependencies = [ "terminal_size", ] +[[package]] +name = "clap_complete" +version = "4.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "037e2a1a92236d0aff7e845093f64661d6df4c02c9fcc61a60e9e1d736fa392f" +dependencies = [ + "clap", +] + [[package]] name = "clap_derive" version = "4.6.7" @@ -357,6 +415,25 @@ version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1c133bc6a41be0d194c306b5506d15e6feeea7b1d6604bd3f8310dfb2ca96486" +[[package]] +name = "clap_mangen" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "211d617eaa4b735c96c9e0228fcbdb5120ef623f2b8cb67ffb84c3e02dbc28a4" +dependencies = [ + "clap", + "roff", +] + +[[package]] +name = "clipboard-win" +version = "5.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "052af7878466feccda12ba7a6a67d019d712f296809559b47c12c8cc196d3a9a" +dependencies = [ + "error-code", +] + [[package]] name = "clru" version = "0.6.3" @@ -366,12 +443,92 @@ dependencies = [ "hashbrown 0.16.1", ] +[[package]] +name = "codesnake" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2205f7f6d3de68ecf4c291c789b3edf07b6569268abd0188819086f71ae42225" + +[[package]] +name = "codspeed" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e7c21807b937b8e535f2745cd83c525fc250524021cd662b9ba4d3e399bb504" +dependencies = [ + "anyhow", + "cc", + "colored", + "getrandom 0.4.3", + "glob", + "nix 0.31.3", + "serde", + "serde_json", + "statrs", +] + +[[package]] +name = "codspeed-divan-compat" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc7e48c4e555e60b894144b14df1e893a7317b9f4000b1492ace3041479b12cc" +dependencies = [ + "clap", + "codspeed", + "codspeed-divan-compat-macros", + "codspeed-divan-compat-walltime", + "regex", +] + +[[package]] +name = "codspeed-divan-compat-macros" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1eb62a7a8d4cc6b79b18dcbac670b3202f4c8c901ca8ea701ab42190f0b14799" +dependencies = [ + "divan-macros", + "itertools 0.14.0", + "proc-macro-crate", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "codspeed-divan-compat-walltime" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15c4e77a97571b6214b976cee7e2a564215d0a5bee9520a3cdab86106d0b31a3" +dependencies = [ + "cfg-if", + "clap", + "codspeed", + "condtype", + "divan-macros", + "libc", + "regex-lite", +] + [[package]] name = "colorchoice" version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" +[[package]] +name = "colored" +version = "3.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "faf9468729b8cbcea668e36183cb69d317348c2e08e994829fb56ebfdfbaac34" +dependencies = [ + "windows-sys 0.59.0", +] + +[[package]] +name = "condtype" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf0a07a401f374238ab8e2f11a104d2851bf9ce711ec69804834de8af45c7af" + [[package]] name = "console" version = "0.16.6" @@ -380,8 +537,8 @@ checksum = "e96a4956774c13c126a8b5af4daa79384f4d826534c95a02d76afb39e2ab64e3" dependencies = [ "encode_unicode", "libc", - "unicode-width", - "windows-sys", + "unicode-width 0.2.3", + "windows-sys 0.61.2", ] [[package]] @@ -505,8 +662,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e0b1fab2ae45819af2d0731d60f2afe17227ebb1a1538a236da84c93e9a60162" dependencies = [ "dispatch2", - "nix", - "windows-sys", + "nix 0.31.3", + "windows-sys 0.61.2", ] [[package]] @@ -589,6 +746,36 @@ dependencies = [ "thiserror", ] +[[package]] +name = "diff" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56254986775e3233ffa9c4d7d3faaf6d36a2c09d30b20687e9f88bc8bafc16c8" + +[[package]] +name = "difflib" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6184e33543162437515c2e2b48714794e37845ec9851711914eec9d308f6ebe8" + +[[package]] +name = "diffutils" +version = "0.6.0" +dependencies = [ + "assert_cmd", + "chrono", + "codspeed-divan-compat", + "diff", + "itoa", + "predicates", + "pretty_assertions", + "rand 0.10.3", + "regex", + "same-file", + "tempfile", + "unicode-width 0.2.3", +] + [[package]] name = "digest" version = "0.10.7" @@ -608,6 +795,27 @@ dependencies = [ "crypto-common 0.2.2", ] +[[package]] +name = "dirs" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3e8aa94d75141228480295a7d0e7feb620b1a5ad9f12bc40be62411e38cce4e" +dependencies = [ + "dirs-sys", +] + +[[package]] +name = "dirs-sys" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e01a3366d27ee9890022452ee61b2b63a67e6f13f58900b651ff5665f0bb1fab" +dependencies = [ + "libc", + "option-ext", + "redox_users", + "windows-sys 0.59.0", +] + [[package]] name = "dispatch2" version = "0.3.1" @@ -631,12 +839,29 @@ dependencies = [ "syn 3.0.9", ] +[[package]] +name = "divan-macros" +version = "0.1.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8dc51d98e636f5e3b0759a39257458b22619cac7e96d932da6eeb052891bb67c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "dunce" version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" +[[package]] +name = "dyn-clone" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" + [[package]] name = "either" version = "1.19.0" @@ -663,6 +888,24 @@ dependencies = [ "simdutf8", ] +[[package]] +name = "encoding_rs_io" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fba3fe847045ecff794b9c138293a80db914678c453ad63fbf0c6a9eb6e00b22" +dependencies = [ + "encoding_rs", +] + +[[package]] +name = "env_logger" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cd405aab171cb85d6735e5c8d9db038c17d3ca007a4d2c25f337935c3d90580" +dependencies = [ + "log", +] + [[package]] name = "equivalent" version = "1.0.3" @@ -676,7 +919,45 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys", + "windows-sys 0.59.0", +] + +[[package]] +name = "error-code" +version = "3.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b5343afd4a8365a643ac588dab4cf234a190c7f6c88c9f6dd6ffe00837661b7" + +[[package]] +name = "etcetera" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de48cc4d1c1d97a20fd819def54b890cadde72ed3ad0c614822a0a433361be96" +dependencies = [ + "cfg-if", + "windows-sys 0.61.2", +] + +[[package]] +name = "faccess" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ae66425802d6a903e268ae1a08b8c38ba143520f227a205edf4e9c7e3e26d5" +dependencies = [ + "bitflags 1.3.2", + "libc", + "winapi", +] + +[[package]] +name = "fancy-regex" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1e1dacd0d2082dfcf1351c4bdd566bbe89a2b263235a2b50058f1e130a47277" +dependencies = [ + "bit-set", + "regex-automata", + "regex-syntax", ] [[package]] @@ -708,6 +989,48 @@ version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" +[[package]] +name = "fd-find" +version = "10.5.1" +dependencies = [ + "aho-corasick", + "anyhow", + "argmax", + "clap", + "clap_complete", + "crossbeam-channel", + "ctrlc", + "diff", + "etcetera", + "faccess", + "filetime", + "globset", + "ignore", + "jiff", + "libc", + "lscolors", + "memchr", + "nix 0.31.3", + "normpath", + "nu-ansi-term", + "regex", + "regex-syntax", + "tempfile", + "test-case", + "tikv-jemallocator", +] + +[[package]] +name = "fd-lock" +version = "4.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ce92ff622d6dadf7349484f42c93271a0d49b7cc4d466a936405bacbe10aa78" +dependencies = [ + "cfg-if", + "rustix", + "windows-sys 0.59.0", +] + [[package]] name = "filetime" version = "0.2.29" @@ -744,6 +1067,15 @@ dependencies = [ "zlib-rs", ] +[[package]] +name = "float-cmp" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b09cf3155332e944990140d967ff5eceb70df778b34f77d8075db46e4704e6d8" +dependencies = [ + "num-traits", +] + [[package]] name = "fluent" version = "0.17.0" @@ -1325,7 +1657,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7518b930f6aeb75e871d32ce873c2ae46e35cef19b5a0b039d4337f651d897cd" dependencies = [ "bstr", - "hashbrown 0.17.1", + "hashbrown 0.16.1", ] [[package]] @@ -1674,7 +2006,7 @@ dependencies = [ "bitflags 2.13.2", "gix-path", "libc", - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -1740,7 +2072,7 @@ dependencies = [ "libc", "parking_lot", "tempfile", - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -1880,6 +2212,110 @@ version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e4eba85ea1d0a966a983acd07deee566e67395d2d96b6fb39e62b5a833f1eb0b" +[[package]] +name = "globset" +version = "0.4.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e47d37d2ae4464254884b60ab7071be2b876a9c35b696bd018ddcc76847309cd" +dependencies = [ + "aho-corasick", + "bstr", + "log", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "grep" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "309217bc53e2c691c314389c7fa91f9cd1a998cda19e25544ea47d94103880c3" +dependencies = [ + "grep-cli", + "grep-matcher", + "grep-pcre2", + "grep-printer", + "grep-regex", + "grep-searcher", +] + +[[package]] +name = "grep-cli" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf32d263c5d5cc2a23ce587097f5ddafdb188492ba2e6fb638eaccdc22453631" +dependencies = [ + "bstr", + "globset", + "libc", + "log", + "termcolor", + "winapi-util", +] + +[[package]] +name = "grep-matcher" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9417543f4870fc8f1c8e1af870afae2431007626d9e703fce6471c468d33847" +dependencies = [ + "memchr", +] + +[[package]] +name = "grep-pcre2" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b752bb6f57dd3716e55b63a33ca05b5825c3d66cd427468874a5564af6ca365c" +dependencies = [ + "grep-matcher", + "log", + "pcre2", +] + +[[package]] +name = "grep-printer" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd76035e87871f51c1ee5b793e32122b3ccf9c692662d9622ef1686ff5321acb" +dependencies = [ + "bstr", + "grep-matcher", + "grep-searcher", + "log", + "serde", + "serde_json", + "termcolor", +] + +[[package]] +name = "grep-regex" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ce0c256c3ad82bcc07b812c15a45ec1d398122e8e15124f96695234db7112ef" +dependencies = [ + "bstr", + "grep-matcher", + "log", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "grep-searcher" +version = "0.1.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72348823a0eafc4bc2e9051064f28b5b42cc100b571b3a35d67918d711efcbc6" +dependencies = [ + "bstr", + "encoding_rs", + "encoding_rs_io", + "grep-matcher", + "log", + "memchr", + "memmap2", +] + [[package]] name = "half" version = "2.7.1" @@ -1953,12 +2389,24 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" +[[package]] +name = "hermit-abi" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e17592d60ebacc7d5e169f4663c5f84f9161cc90328abcfe8456f41e4dfcb284" + [[package]] name = "hex" version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +[[package]] +name = "hifijson" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a7763b98ba8a24f59e698bf9ab197e7676c640d6455d1580b4ce7dc560f0f0d" + [[package]] name = "hostname" version = "0.4.2" @@ -2316,6 +2764,22 @@ dependencies = [ "icu_properties", ] +[[package]] +name = "ignore" +version = "0.4.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b009b6744c1445efd7244084e25e498636412effb6760b55067553baa925cc7" +dependencies = [ + "crossbeam-deque", + "globset", + "log", + "memchr", + "regex-automata", + "same-file", + "walkdir", + "winapi-util", +] + [[package]] name = "indexmap" version = "2.14.2" @@ -2336,7 +2800,7 @@ checksum = "9433806cd6b4ec1aba79c021c7e4c58fb4c3b9977c085062e611ac929998fb0c" dependencies = [ "console", "portable-atomic", - "unicode-width", + "unicode-width 0.2.3", "unit-prefix", "web-time", ] @@ -2380,12 +2844,32 @@ dependencies = [ "unic-langid", ] +[[package]] +name = "is-terminal" +version = "0.4.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" +dependencies = [ + "hermit-abi", + "libc", + "windows-sys 0.59.0", +] + [[package]] name = "is_terminal_polyfill" version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", +] + [[package]] name = "itertools" version = "0.15.0" @@ -2401,6 +2885,67 @@ version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" +[[package]] +name = "jaq" +version = "2.3.0" +dependencies = [ + "codesnake", + "dirs", + "env_logger", + "hifijson", + "is-terminal", + "jaq-core", + "jaq-json", + "jaq-std", + "log", + "memmap2", + "mimalloc", + "rustyline", + "tempfile", + "unicode-width 0.1.14", + "yansi", +] + +[[package]] +name = "jaq-core" +version = "2.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77526a72eb79412c29fd141767a6549bbfcb1cb40e00556fe16532d5e878e098" +dependencies = [ + "dyn-clone", + "once_cell", + "typed-arena", +] + +[[package]] +name = "jaq-json" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01dbdbd07b076e8403abac68ce7744d93e2ecd953bbc44bf77bf00e1e81172bc" +dependencies = [ + "foldhash 0.1.5", + "hifijson", + "indexmap", + "jaq-core", + "jaq-std", +] + +[[package]] +name = "jaq-std" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c264fe397c981705976c71f1bfe020382b9eda52ae950e57fe885e147bdd67d" +dependencies = [ + "aho-corasick", + "base64", + "chrono", + "jaq-core", + "libm", + "log", + "regex-lite", + "urlencoding", +] + [[package]] name = "jiff" version = "0.2.39" @@ -2466,6 +3011,16 @@ dependencies = [ "jiff-tzdb", ] +[[package]] +name = "jobserver" +version = "0.1.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c00acbd29eabad4a2392fa0e921c874934dbbf4194312ad20f04a0ed67a3cb3" +dependencies = [ + "getrandom 0.4.3", + "libc", +] + [[package]] name = "js-sys" version = "0.3.107" @@ -2513,6 +3068,12 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" +[[package]] +name = "lexopt" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "803ec87c9cfb29b9d2633f20cba1f488db3fd53f2158b1024cbefb47ba05d413" + [[package]] name = "libc" version = "0.2.191" @@ -2525,6 +3086,24 @@ version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" +[[package]] +name = "libmimalloc-sys" +version = "0.1.49" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a45a52f43e1c16f667ccfe4dd8c85b7f7c204fd5e3bf46c5b0db9a5c3c0b8e9" +dependencies = [ + "cc", +] + +[[package]] +name = "libredox" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61ff90caf6077a803a240f62fdbe88645a890bbca49ef8174c3cb0404362171d" +dependencies = [ + "libc", +] + [[package]] name = "linux-raw-sys" version = "0.12.1" @@ -2596,6 +3175,15 @@ dependencies = [ "libc", ] +[[package]] +name = "mimalloc" +version = "0.1.52" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d4139bb28d14ad1facf21d5eb8825051b326e172d216b39f6d31df53cc97862" +dependencies = [ + "libmimalloc-sys", +] + [[package]] name = "miniz_oxide" version = "0.9.1" @@ -2615,7 +3203,7 @@ dependencies = [ "libc", "log", "wasi 0.11.1+wasi-snapshot-preview1", - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -2624,6 +3212,29 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "743fb55ba31b18fb1ecef6bdc9aa2743314978ac084044301a7eee33fb99a20d" +[[package]] +name = "nix" +version = "0.27.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2eb04e9c688eff1c89d72b407f168cf79bb9e867a9d3323ed6c01519eb9cc053" +dependencies = [ + "bitflags 2.13.2", + "cfg-if", + "libc", +] + +[[package]] +name = "nix" +version = "0.30.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74523f3a35e05aba87a1d978330aef40f67b0304ac79c1c00b294c9830543db6" +dependencies = [ + "bitflags 2.13.2", + "cfg-if", + "cfg_aliases", + "libc", +] + [[package]] name = "nix" version = "0.31.3" @@ -2651,6 +3262,21 @@ version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9737e026353e5cd0736f98eddae28665118eb6f6600902a7f50db585621fecb6" +[[package]] +name = "normalize-line-endings" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61807f77802ff30975e01f4f071c8ba10c022052f98b3294119f3e615d13e5be" + +[[package]] +name = "normpath" +version = "1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b11ce00d2594068e8a27c9146fdc5cf9f3ac38eb42c7cd34d05fea618873ac9f" +dependencies = [ + "windows-sys 0.61.2", +] + [[package]] name = "notify" version = "8.2.1" @@ -2667,7 +3293,7 @@ dependencies = [ "objc2-core-foundation", "objc2-core-services", "walkdir", - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -2685,7 +3311,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys", + "windows-sys 0.59.0", ] [[package]] @@ -2790,13 +3416,19 @@ version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" +[[package]] +name = "option-ext" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" + [[package]] name = "os_display" version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ad5fd71b79026fb918650dde6d125000a233764f1c2f1659a1c71118e33ea08f" dependencies = [ - "unicode-width", + "unicode-width 0.2.3", ] [[package]] @@ -2839,18 +3471,85 @@ dependencies = [ "winnow", ] +[[package]] +name = "pcre2" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e970b0fcce0c7ee6ef662744ff711f21ccd6f11b7cf03cd187a80e89797fc67" +dependencies = [ + "libc", + "log", + "pcre2-sys", +] + +[[package]] +name = "pcre2-sys" +version = "0.2.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18b9073c1a2549bd409bf4a32c94d903bb1a09bf845bc306ae148897fa0760a4" +dependencies = [ + "cc", + "libc", + "pkg-config", +] + [[package]] name = "percent-encoding" version = "2.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" +[[package]] +name = "phf" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "010378780309880b08997fae13be7834dba947d36393bd372f2b1556deb2a2f6" +dependencies = [ + "phf_shared", + "serde", +] + +[[package]] +name = "phf_codegen" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41b585a510fb76fdebead6897982ef2a03a21d8e6cbcca904999742a4afc6ffe" +dependencies = [ + "phf_generator", + "phf_shared", +] + +[[package]] +name = "phf_generator" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aeb62e0959d5a1bebc965f4d15d9e2b7cea002b6b0f5ba8cde6cc26738467100" +dependencies = [ + "fastrand", + "phf_shared", +] + +[[package]] +name = "phf_shared" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6fd9027e2d9319be6349febd1db4e8d02aa544921200c9b777720ac34a3aa89" +dependencies = [ + "siphasher", +] + [[package]] name = "pin-project-lite" version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" +[[package]] +name = "pkg-config" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548" + [[package]] name = "platform-info" version = "2.2.1" @@ -2858,7 +3557,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aa5c6ef5da99ec17edc36267860ddb03504a916e0cc74c0c4dd1c5cdb19637f8" dependencies = [ "libc", - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -2896,6 +3595,46 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "predicates" +version = "3.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ada8f2932f28a27ee7b70dd6c1c39ea0675c55a36879ab92f3a715eaa1e63cfe" +dependencies = [ + "anstyle", + "difflib", + "float-cmp", + "normalize-line-endings", + "predicates-core", + "regex", +] + +[[package]] +name = "predicates-core" +version = "1.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cad38746f3166b4031b1a0d39ad9f954dd291e7854fcc0eed52ee41a0b50d144" + +[[package]] +name = "predicates-tree" +version = "1.0.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0de1b847b39c8131db0467e9df1ff60e6d0562ab8e9a16e568ad0fdb372e2f2" +dependencies = [ + "predicates-core", + "termtree", +] + +[[package]] +name = "pretty_assertions" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ae130e2f271fbc2ac3a40fb1d07180839cdbbe443c7a27e1e3c13c5cac0116d" +dependencies = [ + "diff", + "yansi", +] + [[package]] name = "prettyplease" version = "0.2.37" @@ -2906,6 +3645,15 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "proc-macro-crate" +version = "3.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e67ba7e9b2b56446f1d419b1d807906278ffa1a658a8a5d8a39dcb1f5a78614f" +dependencies = [ + "toml_edit", +] + [[package]] name = "proc-macro2" version = "1.0.107" @@ -3055,6 +3803,16 @@ dependencies = [ "bitflags 2.13.2", ] +[[package]] +name = "redox_users" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60dc65c0ff1a7ae1294b0c67b9f14baf70b644404010370171787bfac1038fc0" +dependencies = [ + "libredox", + "thiserror", +] + [[package]] name = "regex" version = "1.13.1" @@ -3078,12 +3836,43 @@ dependencies = [ "regex-syntax", ] +[[package]] +name = "regex-lite" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cab834c73d247e67f4fae452806d17d3c7501756d98c8808d7c9c7aa7d18f973" + [[package]] name = "regex-syntax" version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" +[[package]] +name = "ripgrep" +version = "15.2.0" +dependencies = [ + "anyhow", + "bstr", + "grep", + "ignore", + "lexopt", + "log", + "serde", + "serde_derive", + "serde_json", + "termcolor", + "textwrap", + "tikv-jemallocator", + "walkdir", +] + +[[package]] +name = "roff" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "323c417e1d9665a65b263ec744ba09030cfb277e9daa0b018a4ab62e57bc8189" + [[package]] name = "rustc-hash" version = "2.1.3" @@ -3100,7 +3889,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys", + "windows-sys 0.59.0", ] [[package]] @@ -3109,6 +3898,26 @@ version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" +[[package]] +name = "rustyline" +version = "13.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02a2d683a4ac90aeef5b1013933f6d977bd37d51ff3f4dad829d4931a7e6be86" +dependencies = [ + "bitflags 2.13.2", + "cfg-if", + "clipboard-win", + "fd-lock", + "libc", + "log", + "memchr", + "nix 0.27.1", + "unicode-segmentation", + "unicode-width 0.1.14", + "utf8parse", + "winapi", +] + [[package]] name = "same-file" version = "1.0.6" @@ -3208,6 +4017,34 @@ dependencies = [ name = "sausage-cwd-probe" version = "0.1.0" +[[package]] +name = "sausage-exec-tools" +version = "0.1.0" +dependencies = [ + "sausage-spawn", +] + +[[package]] +name = "sausage-findutils" +version = "0.1.0" +dependencies = [ + "globset", + "sausage-spawn", + "shlex 1.3.0", + "walkdir", +] + +[[package]] +name = "sausage-grep" +version = "0.1.0" +dependencies = [ + "globset", + "grep-matcher", + "grep-regex", + "lexopt", + "walkdir", +] + [[package]] name = "sausage-procps" version = "0.1.0" @@ -3215,12 +4052,50 @@ dependencies = [ "wit-bindgen 0.46.0", ] +[[package]] +name = "sausage-sed" +version = "0.1.0" +dependencies = [ + "sed", +] + +[[package]] +name = "sausage-spawn" +version = "0.1.0" +dependencies = [ + "wit-bindgen 0.46.0", +] + [[package]] name = "scopeguard" version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" +[[package]] +name = "sed" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b908cd347ea83fee7c9fdb621e4dccdd6612d9962da2be03a846be2e6d83f758" +dependencies = [ + "clap", + "clap_complete", + "clap_mangen", + "fancy-regex 0.18.0", + "memchr", + "memmap2", + "phf", + "phf_codegen", + "predicates", + "regex", + "rustix", + "tempfile", + "terminal_size", + "textwrap", + "unicode-width 0.2.3", + "uucore", +] + [[package]] name = "self_cell" version = "1.3.0" @@ -3326,6 +4201,12 @@ dependencies = [ "sponge-cursor", ] +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + [[package]] name = "shlex" version = "2.0.1" @@ -3344,6 +4225,12 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" +[[package]] +name = "siphasher" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33f4fe9184a62d842c9ef383018f3306d8ba224fd9d836f56d7288308847c256" + [[package]] name = "slab" version = "0.4.12" @@ -3365,6 +4252,12 @@ version = "1.16.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0d5d4b1981b8bbbb7aed204cae5726d683d5d6ac7e74ec54122ea28f315e7c8" +[[package]] +name = "smawk" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8e2fb0f499abb4d162f2bedad68f5ef91a1682b5a03596ddb67efd37768d100" + [[package]] name = "spin" version = "0.10.1" @@ -3383,6 +4276,16 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" +[[package]] +name = "statrs" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a3fe7c28c6512e766b0874335db33c94ad7b8f9054228ae1c2abd47ce7d335e" +dependencies = [ + "approx", + "num-traits", +] + [[package]] name = "strsim" version = "0.11.1" @@ -3438,7 +4341,16 @@ dependencies = [ "getrandom 0.4.3", "once_cell", "rustix", - "windows-sys", + "windows-sys 0.59.0", +] + +[[package]] +name = "termcolor" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06794f8f6c5c898b3275aebefa6b8a1cb24cd2c6c79397ab15774837a0bc5755" +dependencies = [ + "winapi-util", ] [[package]] @@ -3448,7 +4360,58 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "230a1b821ccbd75b185820a1f1ff7b14d21da1e442e22c0863ea5f08771a8874" dependencies = [ "rustix", - "windows-sys", + "windows-sys 0.59.0", +] + +[[package]] +name = "termtree" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f50febec83f5ee1df3015341d8bd429f2d1cc62bcba7ea2076759d315084683" + +[[package]] +name = "test-case" +version = "3.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb2550dd13afcd286853192af8601920d959b14c401fcece38071d53bf0768a8" +dependencies = [ + "test-case-macros", +] + +[[package]] +name = "test-case-core" +version = "3.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29b8e1f65819001927b8cdcbb5560f2428212f2e5ab5684e3c6815a4d80e72db" +dependencies = [ + "cfg-if", + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "test-case-macros" +version = "3.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "802d6165a14df1598b7290a8c546df402940d3cfbe82d15c0fd35de176771693" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", + "test-case-core", +] + +[[package]] +name = "textwrap" +version = "0.16.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c13547615a44dc9c452a8a534638acdf07120d4b6847c8178705da06306a3057" +dependencies = [ + "smawk", + "terminal_size", + "unicode-linebreak", + "unicode-width 0.2.3", ] [[package]] @@ -3471,6 +4434,26 @@ dependencies = [ "syn 3.0.9", ] +[[package]] +name = "tikv-jemalloc-sys" +version = "0.7.1+5.3.1-0-g81034ce1f1373e37dc865038e1bc8eeecf559ce8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a2825c78386b4ae0314074867860ba9577875de945f05992c38815cbec327f0" +dependencies = [ + "cc", + "libc", +] + +[[package]] +name = "tikv-jemallocator" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "249f09e49ab1609436f34c776e84231bead18d6a955f119f939bdc1d847561bd" +dependencies = [ + "libc", + "tikv-jemalloc-sys", +] + [[package]] name = "tinystr" version = "0.8.5" @@ -3488,6 +4471,36 @@ version = "1.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fd3ca314f692efd6c868f8408f53fe444634a845f96c028b97d35f6a1f79f0ee" +[[package]] +name = "toml_datetime" +version = "1.1.3+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "173f25c883cd2176a2d8212155638c0a60d484a9d84546d270866f40f56b6e49" +dependencies = [ + "serde_core", +] + +[[package]] +name = "toml_edit" +version = "0.25.19+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd43688c4858d7264ae1cf9a9f874a67c3feb74a883b1cd7bebd6a4ea8307218" +dependencies = [ + "indexmap", + "toml_datetime", + "toml_parser", + "winnow", +] + +[[package]] +name = "toml_parser" +version = "1.1.5+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baa693a8032d7e1cada7d0041e96126df243179ff061456783ac7f12bda4744c" +dependencies = [ + "winnow", +] + [[package]] name = "type-map" version = "0.5.1" @@ -3497,6 +4510,12 @@ dependencies = [ "rustc-hash", ] +[[package]] +name = "typed-arena" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6af6ae20167a9ece4bcb41af5b80f8a1f1df981f6391189ce00fd257af04126a" + [[package]] name = "typenum" version = "1.20.1" @@ -3533,6 +4552,12 @@ version = "1.0.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2c754d6c33795a1c324727428e5a7dedb5b06195f9890bdbcba760d3e246563" +[[package]] +name = "unicode-linebreak" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b09c83c3c29d37506a3e260c08c03743a6bb66a9cd432c6934ab501a190571f" + [[package]] name = "unicode-normalization" version = "0.1.26" @@ -3542,6 +4567,18 @@ dependencies = [ "tinyvec", ] +[[package]] +name = "unicode-segmentation" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493" + +[[package]] +name = "unicode-width" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af" + [[package]] name = "unicode-width" version = "0.2.3" @@ -3572,6 +4609,12 @@ dependencies = [ "serde", ] +[[package]] +name = "urlencoding" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" + [[package]] name = "utf16_iter" version = "1.1.0" @@ -3685,7 +4728,7 @@ dependencies = [ "rustix", "thiserror", "uucore", - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -3733,12 +4776,12 @@ dependencies = [ "fluent", "indicatif", "libc", - "nix", + "nix 0.31.3", "rustix", "thiserror", "uucore", "walkdir", - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -3785,7 +4828,7 @@ dependencies = [ "rustix", "thiserror", "uucore", - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -3867,7 +4910,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "601663e752b83657be04ec8c54838310a516dd66d30affc36f02f3cc71f26199" dependencies = [ "clap", - "fancy-regex", + "fancy-regex 0.19.2", "fluent", "num-bigint", "num-traits", @@ -3920,7 +4963,7 @@ checksum = "b28755578b0b98318371e055fcc70b94c9198bacfce0d125d3049ab1c2d2beb2" dependencies = [ "clap", "fluent", - "unicode-width", + "unicode-width 0.2.3", "uucore", ] @@ -4054,12 +5097,12 @@ dependencies = [ "fluent", "indicatif", "libc", - "nix", + "nix 0.31.3", "rustc-hash", "rustix", "thiserror", "uucore", - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -4072,7 +5115,7 @@ dependencies = [ "fluent", "rustix", "uucore", - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -4159,7 +5202,7 @@ checksum = "a86126634ad64f323a8e2d487fc7fa3b92b1b251a08eef631c113ae57d112678" dependencies = [ "clap", "fluent", - "itertools", + "itertools 0.15.0", "memchr", "regex", "thiserror", @@ -4387,7 +5430,7 @@ dependencies = [ "ctrlc", "fluent", "foldhash 0.2.0", - "itertools", + "itertools 0.15.0", "libc", "memchr", "rand 0.10.3", @@ -4438,7 +5481,7 @@ dependencies = [ "notify", "rustix", "uucore", - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -4465,7 +5508,7 @@ dependencies = [ "rustix", "thiserror", "uucore", - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -4483,7 +5526,7 @@ dependencies = [ "rustix", "thiserror", "uucore", - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -4535,7 +5578,7 @@ dependencies = [ "rustix", "thiserror", "uucore", - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -4548,7 +5591,7 @@ dependencies = [ "fluent", "rustix", "uucore", - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -4621,7 +5664,7 @@ dependencies = [ "libc", "rustix", "thiserror", - "unicode-width", + "unicode-width 0.2.3", "uucore", ] @@ -4633,7 +5676,7 @@ checksum = "7a00721c606207353591d7d5a4fe6c89d94073decccd061392f3f0dea1c37af4" dependencies = [ "clap", "fluent", - "itertools", + "itertools 0.15.0", "rustix", "uucore", ] @@ -4644,6 +5687,7 @@ version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f24a5910ebbf2c0baf486a9125705f8e026a8f7741a3ea8cf5eca3c9257c9bdf" dependencies = [ + "ariadne", "base64-simd", "bigdecimal", "blake2b_simd", @@ -4665,13 +5709,13 @@ dependencies = [ "icu_decimal", "icu_locale", "icu_provider", - "itertools", + "itertools 0.15.0", "jiff", "jiff-icu", "libc", "md-5", "memchr", - "nix", + "nix 0.31.3", "num-traits", "os_display", "procfs", @@ -4684,12 +5728,12 @@ dependencies = [ "sm3", "thiserror", "unic-langid", - "unicode-width", + "unicode-width 0.2.3", "unit-prefix", "uucore_procs", "walkdir", "wild", - "windows-sys", + "windows-sys 0.61.2", "xattr", "z85", ] @@ -4725,6 +5769,15 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5c3082ca00d5a5ef149bb8b555a72ae84c9c59f7250f013ac822ac2e49b19c64" +[[package]] +name = "wait-timeout" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ac3b126d3914f9849036f826e054cbabdc8519970b8998ddaf3b5bd3c65f11" +dependencies = [ + "libc", +] + [[package]] name = "walkdir" version = "2.5.0" @@ -4856,15 +5909,37 @@ dependencies = [ "glob", ] +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + [[package]] name = "winapi-util" version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys", + "windows-sys 0.59.0", ] +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + [[package]] name = "windows-core" version = "0.62.2" @@ -4924,6 +5999,15 @@ dependencies = [ "windows-link", ] +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets", +] + [[package]] name = "windows-sys" version = "0.61.2" @@ -4933,6 +6017,70 @@ dependencies = [ "windows-link", ] +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + [[package]] name = "winnow" version = "1.0.5" @@ -5076,6 +6224,12 @@ dependencies = [ "rustix", ] +[[package]] +name = "yansi" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfe53a6657fd280eaa890a3bc59152892ffa3e30101319d168b781ed6529b049" + [[package]] name = "yoke" version = "0.8.4" diff --git a/userland/Cargo.toml b/userland/Cargo.toml index a259dbc..271c59c 100644 --- a/userland/Cargo.toml +++ b/userland/Cargo.toml @@ -1,7 +1,7 @@ # The default CLIs of a world, built for wasm32-wasip2. Its own workspace, outside the host's. [workspace] resolver = "3" -members = ["git", "coreutils", "procps", "cwd-init/probe"] +members = ["git", "coreutils", "procps", "cwd-init/probe", "ripgrep", "diffutils", "jaq", "sed", "grep", "fd", "spawn", "findutils", "exec-tools"] # WASI compatibility patches: git/vendor/PATCHES.md. [patch.crates-io] diff --git a/userland/diffutils/.cargo-ok b/userland/diffutils/.cargo-ok new file mode 100644 index 0000000..5f8b795 --- /dev/null +++ b/userland/diffutils/.cargo-ok @@ -0,0 +1 @@ +{"v":1} \ No newline at end of file diff --git a/userland/diffutils/.cargo_vcs_info.json b/userland/diffutils/.cargo_vcs_info.json new file mode 100644 index 0000000..9d2bc76 --- /dev/null +++ b/userland/diffutils/.cargo_vcs_info.json @@ -0,0 +1,6 @@ +{ + "git": { + "sha1": "815ed7d8de89a236e5f636749f80a5b061017521" + }, + "path_in_vcs": "" +} \ No newline at end of file diff --git a/userland/diffutils/.github/workflows/GnuComment.yml b/userland/diffutils/.github/workflows/GnuComment.yml new file mode 100644 index 0000000..c654ddb --- /dev/null +++ b/userland/diffutils/.github/workflows/GnuComment.yml @@ -0,0 +1,100 @@ +name: GnuComment + +on: + workflow_run: + workflows: ["GnuTests"] + types: + - completed + +permissions: {} + +jobs: + post-comment: + permissions: + actions: read # to list workflow runs artifacts + pull-requests: write # to comment on pr + + runs-on: ubuntu-latest + if: > + github.event.workflow_run.event == 'pull_request' + steps: + - name: 'Download artifact' + uses: actions/github-script@v9 + with: + script: | + // List all artifacts from GnuTests + var artifacts = await github.rest.actions.listWorkflowRunArtifacts({ + owner: context.repo.owner, + repo: context.repo.repo, + run_id: ${{ github.event.workflow_run.id }}, + }); + + // Download the "comment" artifact, which contains a PR number (NR) and result.txt + var matchArtifact = artifacts.data.artifacts.filter((artifact) => { + return artifact.name == "comment" + })[0]; + + if (!matchArtifact) { + console.log('No comment artifact found'); + return; + } + + var download = await github.rest.actions.downloadArtifact({ + owner: context.repo.owner, + repo: context.repo.repo, + artifact_id: matchArtifact.id, + archive_format: 'zip', + }); + var fs = require('fs'); + fs.writeFileSync('${{ github.workspace }}/comment.zip', Buffer.from(download.data)); + + - run: unzip comment.zip || echo "Failed to unzip comment artifact" + + - name: 'Comment on PR' + uses: actions/github-script@v9 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + var fs = require('fs'); + + // Check if files exist + if (!fs.existsSync('./NR')) { + console.log('No NR file found, skipping comment'); + return; + } + if (!fs.existsSync('./result.txt')) { + console.log('No result.txt file found, skipping comment'); + return; + } + + var issue_number = Number(fs.readFileSync('./NR')); + var content = fs.readFileSync('./result.txt'); + + if (content.toString().trim().length > 7) { // 7 because we have backquote + \n + // Update existing comment if present, otherwise create a new one + var marker = ''; + var body = marker + '\nGNU diffutils testsuite comparison:\n```\n' + content + '```'; + var comments = await github.rest.issues.listComments({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: issue_number, + }); + var existing = comments.data.filter(c => c.body.includes(marker))[0]; + if (existing) { + await github.rest.issues.updateComment({ + owner: context.repo.owner, + repo: context.repo.repo, + comment_id: existing.id, + body: body, + }); + } else { + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: issue_number, + body: body, + }); + } + } else { + console.log('Comment content too short, skipping'); + } diff --git a/userland/diffutils/.github/workflows/GnuTests.yml b/userland/diffutils/.github/workflows/GnuTests.yml new file mode 100644 index 0000000..9880012 --- /dev/null +++ b/userland/diffutils/.github/workflows/GnuTests.yml @@ -0,0 +1,257 @@ +name: GnuTests + +# Run GNU diffutils testsuite against the Rust diffutils implementation +# and compare results against the main branch to catch regressions + +on: + pull_request: + push: + branches: + - '*' + +permissions: + contents: write # Publish diffutils instead of discarding + +# End the current execution if there is a new changeset in the PR +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} + +env: + DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} + TEST_FULL_SUMMARY_FILE: 'diffutils-gnu-full-result.json' + +jobs: + native: + name: Run GNU diffutils testsuite + runs-on: ubuntu-24.04 + steps: + - name: Checkout code + uses: actions/checkout@v7 + with: + persist-credentials: false + + - uses: Swatinem/rust-cache@v2 + + ### Build + - name: Build Rust diffutils binary + shell: bash + run: | + ## Build Rust diffutils binary + cargo build --config=profile.release.strip=true --profile=release + zstd -19 target/release/diffutils -o diffutils-x86_64-unknown-linux-gnu.zst + + - name: Publish latest commit + uses: softprops/action-gh-release@v3 + if: github.event_name == 'push' && github.ref == 'refs/heads/main' + with: + tag_name: latest-commit + body: | + commit: ${{ github.sha }} + draft: false + prerelease: true + files: | + diffutils-x86_64-unknown-linux-gnu.zst + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + ### Run tests + - name: Run GNU diffutils testsuite + shell: bash + run: | + ## Run GNU diffutils testsuite + # Exit code 1 means some tests failed, which is expected and handled + # by the comparison below; 2 means the suite couldn't be run at all + # 'shell: bash' runs with -e, so don't let the expected exit code 1 + # abort the step before the exit code is inspected + result=0 + ./tests/run-upstream-testsuite.sh release || result=$? + if [[ $result -ge 2 ]]; then + echo "::error ::The GNU testsuite could not be run (exit code $result); see the log above" + exit 1 + fi + env: + TERM: xterm + + - name: Upload full json results + uses: actions/upload-artifact@v7 + with: + name: diffutils-gnu-full-result + path: tests/test-results.json + if-no-files-found: warn + + aggregate: + needs: [native] + permissions: + actions: read + contents: read + pull-requests: read + name: Aggregate GNU test results + runs-on: ubuntu-24.04 + steps: + - name: Initialize workflow variables + id: vars + shell: bash + run: | + ## VARs setup + outputs() { step_id="${{ github.action }}"; for var in "$@" ; do echo steps.${step_id}.outputs.${var}="${!var}"; echo "${var}=${!var}" >> $GITHUB_OUTPUT; done; } + + TEST_SUMMARY_FILE='diffutils-gnu-result.json' + outputs TEST_SUMMARY_FILE + + - name: Checkout code + uses: actions/checkout@v7 + with: + persist-credentials: false + + - name: Retrieve reference artifacts + uses: dawidd6/action-download-artifact@v27 + continue-on-error: true + with: + workflow: GnuTests.yml + branch: "${{ env.DEFAULT_BRANCH }}" + workflow_conclusion: completed + path: "reference" + if_no_artifact_found: warn + + - name: Download full json results + uses: actions/download-artifact@v8 + with: + name: diffutils-gnu-full-result + path: results + + - name: Extract/summarize testing info + id: summary + shell: bash + run: | + ## Extract/summarize testing info + outputs() { step_id="${{ github.action }}"; for var in "$@" ; do echo steps.${step_id}.outputs.${var}="${!var}"; echo "${var}=${!var}" >> $GITHUB_OUTPUT; done; } + + RESULT_FILE="results/test-results.json" + if [[ ! -f "$RESULT_FILE" ]]; then + echo "::error ::Missing test results at $RESULT_FILE" + exit 1 + fi + + if ! jq -e . "$RESULT_FILE" > /dev/null; then + echo "::error ::Test results at $RESULT_FILE are not valid JSON" + exit 1 + fi + + TOTAL=$(jq '[.tests[]] | length' "$RESULT_FILE") + PASS=$(jq '[.tests[] | select(.result=="PASS")] | length' "$RESULT_FILE") + FAIL=$(jq '[.tests[] | select(.result=="FAIL")] | length' "$RESULT_FILE") + SKIP=$(jq '[.tests[] | select(.result=="SKIP")] | length' "$RESULT_FILE") + ERROR=0 + + if [[ "$TOTAL" -eq 0 ]]; then + echo "::error ::No test was run; refusing to report or compare an empty test run" + exit 1 + fi + + output="GNU diffutils tests summary = TOTAL: $TOTAL / PASS: $PASS / FAIL: $FAIL / SKIP: $SKIP" + echo "${output}" + + if [[ "$FAIL" -gt 0 ]]; then + echo "::warning ::${output}" + fi + + jq -n \ + --arg date "$(date --rfc-email)" \ + --arg sha "$GITHUB_SHA" \ + --arg total "$TOTAL" \ + --arg pass "$PASS" \ + --arg skip "$SKIP" \ + --arg fail "$FAIL" \ + --arg error "$ERROR" \ + '{($date): { sha: $sha, total: $total, pass: $pass, skip: $skip, fail: $fail, error: $error }}' > '${{ steps.vars.outputs.TEST_SUMMARY_FILE }}' + + HASH=$(sha1sum '${{ steps.vars.outputs.TEST_SUMMARY_FILE }}' | cut --delim=" " -f 1) + outputs HASH TOTAL PASS FAIL SKIP + + - name: Upload SHA1/ID of 'test-summary' + uses: actions/upload-artifact@v7 + with: + name: "${{ steps.summary.outputs.HASH }}" + path: "${{ steps.vars.outputs.TEST_SUMMARY_FILE }}" + + - name: Upload test results summary + uses: actions/upload-artifact@v7 + with: + name: test-summary + path: "${{ steps.vars.outputs.TEST_SUMMARY_FILE }}" + + - name: Compare test failures VS reference + shell: bash + run: | + ## Compare test failures VS reference + REF_SUMMARY_FILE='reference/diffutils-gnu-full-result/test-results.json' + CURRENT_SUMMARY_FILE="results/test-results.json" + + IGNORE_INTERMITTENT=".github/workflows/ignore-intermittent.txt" + + # Build the comment in a directory of its own: 'reference/' holds the + # artifacts downloaded from the reference run, including its own + # 'comment' artifact, and re-uploading those would post a comparison + # belonging to another run (see https://github.com/uutils/diffutils/pull/262) + COMMENT_DIR="comment" + rm -rf ${COMMENT_DIR} + mkdir -p ${COMMENT_DIR} + echo ${{ github.event.number }} > ${COMMENT_DIR}/NR + COMMENT_LOG="${COMMENT_DIR}/result.txt" + : > "${COMMENT_LOG}" + + COMPARISON_RESULT=0 + if test -f "${CURRENT_SUMMARY_FILE}"; then + if test -s "${REF_SUMMARY_FILE}"; then + echo "Reference summary SHA1/ID: $(sha1sum -- "${REF_SUMMARY_FILE}")" + echo "Current summary SHA1/ID: $(sha1sum -- "${CURRENT_SUMMARY_FILE}")" + + python3 util/compare_test_results.py \ + --ignore-file "${IGNORE_INTERMITTENT}" \ + --output "${COMMENT_LOG}" \ + "${CURRENT_SUMMARY_FILE}" "${REF_SUMMARY_FILE}" + + COMPARISON_RESULT=$? + else + echo "::warning ::Skipping test comparison; no usable reference summary is available at '${REF_SUMMARY_FILE}'." + fi + else + echo "::error ::Failed to find summary of test results (missing '${CURRENT_SUMMARY_FILE}'); failing early" + exit 1 + fi + + if [ ${COMPARISON_RESULT} -eq 1 ]; then + echo "::error ::Found new non-intermittent test failures" + exit 1 + elif [ ${COMPARISON_RESULT} -ge 2 ]; then + # The comparison itself failed (e.g. an unusable reference summary). + # Don't post a comment rather than post a misleading one. + : > "${COMMENT_LOG}" + echo "::warning ::Could not compare the test results against the reference" + else + echo "::notice ::No new test failures detected" + fi + + - name: Upload comparison log (for GnuComment workflow) + if: success() || failure() + uses: actions/upload-artifact@v7 + with: + name: comment + path: comment/ + + - name: Report test results + if: success() || failure() + shell: bash + run: | + ## Report final results + echo "::notice ::GNU diffutils testsuite results:" + echo "::notice :: Total tests: ${{ steps.summary.outputs.TOTAL }}" + echo "::notice :: Passed: ${{ steps.summary.outputs.PASS }}" + echo "::notice :: Failed: ${{ steps.summary.outputs.FAIL }}" + echo "::notice :: Skipped: ${{ steps.summary.outputs.SKIP }}" + + if [[ "${{ steps.summary.outputs.FAIL }}" -gt 0 ]]; then + PASS_RATE=$(( ${{ steps.summary.outputs.PASS }} * 100 / (${{ steps.summary.outputs.PASS }} + ${{ steps.summary.outputs.FAIL }}) )) + echo "::notice :: Pass rate: ${PASS_RATE}%" + fi diff --git a/userland/diffutils/.github/workflows/audit.yml b/userland/diffutils/.github/workflows/audit.yml new file mode 100644 index 0000000..1b8cd61 --- /dev/null +++ b/userland/diffutils/.github/workflows/audit.yml @@ -0,0 +1,15 @@ +name: Security audit + +on: + schedule: + - cron: "0 0 * * *" +jobs: + audit: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + with: + persist-credentials: false + - uses: rustsec/audit-check@v2 + with: + token: ${{ secrets.GITHUB_TOKEN }} diff --git a/userland/diffutils/.github/workflows/ci.yml b/userland/diffutils/.github/workflows/ci.yml new file mode 100644 index 0000000..c441157 --- /dev/null +++ b/userland/diffutils/.github/workflows/ci.yml @@ -0,0 +1,137 @@ +on: [push, pull_request] + +name: Basic CI + +env: + CARGO_TERM_COLOR: always + CARGO_INCREMENTAL: 0 + +jobs: + check: + name: cargo check + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macOS-latest, windows-latest] + steps: + - uses: actions/checkout@v7 + - run: cargo check + + test: + name: cargo test + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macOS-latest, windows-latest] + steps: + - uses: actions/checkout@v7 + - name: install GNU patch on MacOS + if: runner.os == 'macOS' + run: | + brew install gpatch + - name: set up PATH on Windows + # Needed to use GNU's patch.exe instead of Strawberry Perl patch + if: runner.os == 'Windows' + run: echo "C:\Program Files\Git\usr\bin" >> $env:GITHUB_PATH + - run: cargo test + + fmt: + name: cargo fmt --all -- --check + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - run: cargo fmt --all -- --check + + clippy: + name: cargo clippy -- -D warnings + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macOS-latest, windows-latest] + steps: + - uses: actions/checkout@v7 + - run: cargo clippy -- -D warnings + + coverage: + name: Code Coverage + env: + RUSTC_BOOTSTRAP: 1 + runs-on: ${{ matrix.job.os }} + strategy: + fail-fast: false + matrix: + job: + - { os: ubuntu-latest , features: unix } + - { os: macos-latest , features: macos } + - { os: windows-latest , features: windows } + steps: + - uses: actions/checkout@v7 + - name: Initialize workflow variables + id: vars + shell: bash + run: | + ## VARs setup + outputs() { step_id="vars"; for var in "$@" ; do echo steps.${step_id}.outputs.${var}="${!var}"; echo "${var}=${!var}" >> $GITHUB_OUTPUT; done; } + # target-specific options + # * CODECOV_FLAGS + CODECOV_FLAGS=$( echo "${{ matrix.job.os }}" | sed 's/[^[:alnum:]]/_/g' ) + outputs CODECOV_FLAGS + + - name: install GNU patch on MacOS + if: runner.os == 'macOS' + run: | + brew install gpatch + - name: set up PATH on Windows + # Needed to use GNU's patch.exe instead of Strawberry Perl patch + if: runner.os == 'Windows' + run: echo "C:\Program Files\Git\usr\bin" >> $env:GITHUB_PATH + - name: Test + run: cargo test --all-features --no-fail-fast + env: + RUSTC_WRAPPER: "" + RUSTFLAGS: "-Cinstrument-coverage -Zcoverage-options=branch -Ccodegen-units=1 -Copt-level=0 -Coverflow-checks=off -Zpanic_abort_tests -Cpanic=abort" + RUSTDOCFLAGS: "-Cpanic=abort" + LLVM_PROFILE_FILE: "diffutils-%p-%m.profraw" + - name: "`grcov` ~ install" + id: build_grcov + shell: bash + run: | + git clone --depth=1 https://github.com/mozilla/grcov.git ~/grcov/ + cd ~/grcov + # Hardcode the version of crossbeam-epoch. See + # https://github.com/uutils/coreutils/issues/3680 + sed -i -e "s|tempfile =|crossbeam-epoch = \"=0.9.8\"\ntempfile =|" Cargo.toml + cargo install --path . + cd - +# Uncomment when the upstream issue +# https://github.com/mozilla/grcov/issues/849 is fixed +# uses: actions-rs/install@v0.1 +# with: +# crate: grcov +# version: latest +# use-tool-cache: false + - name: Generate coverage data (via `grcov`) + id: coverage + shell: bash + run: | + ## Generate coverage data + COVERAGE_REPORT_DIR="target/debug" + COVERAGE_REPORT_FILE="${COVERAGE_REPORT_DIR}/lcov.info" + mkdir -p "${COVERAGE_REPORT_DIR}" + # display coverage files + grcov . --output-type files --binary-path "${COVERAGE_REPORT_DIR}" | sort --unique + # generate coverage report + grcov . --output-type lcov --output-path "${COVERAGE_REPORT_FILE}" --binary-path "${COVERAGE_REPORT_DIR}" --branch + echo "report=${COVERAGE_REPORT_FILE}" >> $GITHUB_OUTPUT + - name: Upload coverage results (to Codecov.io) + uses: codecov/codecov-action@v7 + with: + token: ${{ secrets.CODECOV_TOKEN }} + files: ${{ steps.coverage.outputs.report }} + ## flags: IntegrationTests, UnitTests, ${{ steps.vars.outputs.CODECOV_FLAGS }} + flags: ${{ steps.vars.outputs.CODECOV_FLAGS }} + name: codecov-umbrella + fail_ci_if_error: false diff --git a/userland/diffutils/.github/workflows/codspeed.yml b/userland/diffutils/.github/workflows/codspeed.yml new file mode 100644 index 0000000..2c25f97 --- /dev/null +++ b/userland/diffutils/.github/workflows/codspeed.yml @@ -0,0 +1,37 @@ +name: CodSpeed + +on: + push: + branches: + - "main" + pull_request: + # `workflow_dispatch` allows CodSpeed to trigger backtest + # performance analysis in order to generate initial data. + workflow_dispatch: + +permissions: + contents: read + id-token: write + +jobs: + codspeed: + name: Run benchmarks + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + + - name: Setup rust toolchain, cache and cargo-codspeed binary + uses: moonrepo/setup-rust@v1 + with: + channel: stable + cache-target: release + bins: cargo-codspeed + + - name: Build the benchmark target(s) + run: cargo codspeed build -m simulation + + - name: Run the benchmarks + uses: CodSpeedHQ/action@v5 + with: + mode: simulation + run: cargo codspeed run diff --git a/userland/diffutils/.github/workflows/fuzzing.yml b/userland/diffutils/.github/workflows/fuzzing.yml new file mode 100644 index 0000000..a218146 --- /dev/null +++ b/userland/diffutils/.github/workflows/fuzzing.yml @@ -0,0 +1,82 @@ +name: Fuzzing + +# spell-checker:ignore fuzzer + +env: + CARGO_INCREMENTAL: 0 + RUSTC_BOOTSTRAP: 1 + +on: + pull_request: + push: + branches: + - main + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} + +jobs: + fuzz-build: + name: Build the fuzzers + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - name: Install `cargo-fuzz` + uses: taiki-e/install-action@v2 + with: + tool: cargo-fuzz + - uses: Swatinem/rust-cache@v2 + with: + shared-key: "cargo-fuzz-cache-key" + cache-directories: "fuzz/target" + - name: Run `cargo-fuzz build` + # https://github.com/rust-fuzz/cargo-fuzz/issues/398 + run: cargo fuzz build --target host-tuple + + fuzz-run: + needs: fuzz-build + name: Run the fuzzers + runs-on: ubuntu-latest + timeout-minutes: 5 + env: + RUN_FOR: 60 + strategy: + matrix: + test-target: + - { name: fuzz_cmp, should_pass: true } + - { name: fuzz_cmp_args, should_pass: true } + - { name: fuzz_ed, should_pass: true } + - { name: fuzz_normal, should_pass: true } + - { name: fuzz_patch, should_pass: true } + - { name: fuzz_side, should_pass: true } + steps: + - uses: actions/checkout@v7 + - name: Install `cargo-fuzz` + uses: taiki-e/install-action@v2 + with: + tool: cargo-fuzz + - uses: Swatinem/rust-cache@v2 + with: + shared-key: "cargo-fuzz-cache-key" + cache-directories: "fuzz/target" + - name: Restore Cached Corpus + uses: actions/cache/restore@v6 + with: + key: corpus-cache-${{ matrix.test-target.name }} + path: | + fuzz/corpus/${{ matrix.test-target.name }} + - name: Run ${{ matrix.test-target.name }} for XX seconds + shell: bash + continue-on-error: ${{ !matrix.test-target.should_pass }} + run: | + cargo fuzz run --target host-tuple ${{ matrix.test-target.name }} -- -max_total_time=${{ env.RUN_FOR }} -detect_leaks=0 + - name: Save Corpus Cache + uses: actions/cache/save@v6 + with: + key: corpus-cache-${{ matrix.test-target.name }} + path: | + fuzz/corpus/${{ matrix.test-target.name }} diff --git a/userland/diffutils/.github/workflows/release.yml b/userland/diffutils/.github/workflows/release.yml new file mode 100644 index 0000000..189549c --- /dev/null +++ b/userland/diffutils/.github/workflows/release.yml @@ -0,0 +1,296 @@ +# This file was autogenerated by dist: https://axodotdev.github.io/cargo-dist +# +# Copyright 2022-2024, axodotdev +# SPDX-License-Identifier: MIT or Apache-2.0 +# +# CI that: +# +# * checks for a Git Tag that looks like a release +# * builds artifacts with dist (archives, installers, hashes) +# * uploads those artifacts to temporary workflow zip +# * on success, uploads the artifacts to a GitHub Release +# +# Note that the GitHub Release will be created with a generated +# title/body based on your changelogs. + +name: Release +permissions: + "contents": "write" + +# This task will run whenever you push a git tag that looks like a version +# like "1.0.0", "v0.1.0-prerelease.1", "my-app/0.1.0", "releases/v1.0.0", etc. +# Various formats will be parsed into a VERSION and an optional PACKAGE_NAME, where +# PACKAGE_NAME must be the name of a Cargo package in your workspace, and VERSION +# must be a Cargo-style SemVer Version (must have at least major.minor.patch). +# +# If PACKAGE_NAME is specified, then the announcement will be for that +# package (erroring out if it doesn't have the given version or isn't dist-able). +# +# If PACKAGE_NAME isn't specified, then the announcement will be for all +# (dist-able) packages in the workspace with that version (this mode is +# intended for workspaces with only one dist-able package, or with all dist-able +# packages versioned/released in lockstep). +# +# If you push multiple tags at once, separate instances of this workflow will +# spin up, creating an independent announcement for each one. However, GitHub +# will hard limit this to 3 tags per commit, as it will assume more tags is a +# mistake. +# +# If there's a prerelease-style suffix to the version, then the release(s) +# will be marked as a prerelease. +on: + pull_request: + push: + tags: + - '**[0-9]+.[0-9]+.[0-9]+*' + +jobs: + # Run 'dist plan' (or host) to determine what tasks we need to do + plan: + runs-on: "ubuntu-22.04" + outputs: + val: ${{ steps.plan.outputs.manifest }} + tag: ${{ !github.event.pull_request && github.ref_name || '' }} + tag-flag: ${{ !github.event.pull_request && format('--tag={0}', github.ref_name) || '' }} + publishing: ${{ !github.event.pull_request }} + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + steps: + - uses: actions/checkout@v7 + with: + persist-credentials: false + submodules: recursive + - name: Install dist + # we specify bash to get pipefail; it guards against the `curl` command + # failing. otherwise `sh` won't catch that `curl` returned non-0 + shell: bash + run: "curl --proto '=https' --tlsv1.2 -LsSf https://github.com/axodotdev/cargo-dist/releases/download/v0.33.0/cargo-dist-installer.sh | sh" + - name: Cache dist + uses: actions/upload-artifact@v7 + with: + name: cargo-dist-cache + path: ~/.cargo/bin/dist + # sure would be cool if github gave us proper conditionals... + # so here's a doubly-nested ternary-via-truthiness to try to provide the best possible + # functionality based on whether this is a pull_request, and whether it's from a fork. + # (PRs run on the *source* but secrets are usually on the *target* -- that's *good* + # but also really annoying to build CI around when it needs secrets to work right.) + - id: plan + run: | + dist ${{ (!github.event.pull_request && format('host --steps=create --tag={0}', github.ref_name)) || 'plan' }} --output-format=json > plan-dist-manifest.json + echo "dist ran successfully" + cat plan-dist-manifest.json + echo "manifest=$(jq -c "." plan-dist-manifest.json)" >> "$GITHUB_OUTPUT" + - name: "Upload dist-manifest.json" + uses: actions/upload-artifact@v7 + with: + name: artifacts-plan-dist-manifest + path: plan-dist-manifest.json + + # Build and packages all the platform-specific things + build-local-artifacts: + name: build-local-artifacts (${{ join(matrix.targets, ', ') }}) + # Let the initial task tell us to not run (currently very blunt) + needs: + - plan + if: ${{ fromJson(needs.plan.outputs.val).ci.github.artifacts_matrix.include != null && (needs.plan.outputs.publishing == 'true' || fromJson(needs.plan.outputs.val).ci.github.pr_run_mode == 'upload') }} + strategy: + fail-fast: false + # Target platforms/runners are computed by dist in create-release. + # Each member of the matrix has the following arguments: + # + # - runner: the github runner + # - dist-args: cli flags to pass to dist + # - install-dist: expression to run to install dist on the runner + # + # Typically there will be: + # - 1 "global" task that builds universal installers + # - N "local" tasks that build each platform's binaries and platform-specific installers + matrix: ${{ fromJson(needs.plan.outputs.val).ci.github.artifacts_matrix }} + runs-on: ${{ matrix.runner }} + container: ${{ matrix.container && matrix.container.image || null }} + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + BUILD_MANIFEST_NAME: target/distrib/${{ join(matrix.targets, '-') }}-dist-manifest.json + steps: + - name: enable windows longpaths + run: | + git config --global core.longpaths true + - uses: actions/checkout@v7 + with: + persist-credentials: false + submodules: recursive + - name: Install Rust non-interactively if not already installed + if: ${{ matrix.container }} + run: | + if ! command -v cargo > /dev/null 2>&1; then + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y + echo "$HOME/.cargo/bin" >> $GITHUB_PATH + fi + - name: Install dist + run: ${{ matrix.install_dist.run }} + # Get the dist-manifest + - name: Fetch local artifacts + uses: actions/download-artifact@v8 + with: + pattern: artifacts-* + path: target/distrib/ + merge-multiple: true + - name: Install dependencies + run: | + ${{ matrix.packages_install }} + - name: Build artifacts + run: | + # Actually do builds and make zips and whatnot + dist build ${{ needs.plan.outputs.tag-flag }} --print=linkage --output-format=json ${{ matrix.dist_args }} > dist-manifest.json + echo "dist ran successfully" + - id: cargo-dist + name: Post-build + # We force bash here just because github makes it really hard to get values up + # to "real" actions without writing to env-vars, and writing to env-vars has + # inconsistent syntax between shell and powershell. + shell: bash + run: | + # Parse out what we just built and upload it to scratch storage + echo "paths<> "$GITHUB_OUTPUT" + dist print-upload-files-from-manifest --manifest dist-manifest.json >> "$GITHUB_OUTPUT" + echo "EOF" >> "$GITHUB_OUTPUT" + + cp dist-manifest.json "$BUILD_MANIFEST_NAME" + - name: "Upload artifacts" + uses: actions/upload-artifact@v7 + with: + name: artifacts-build-local-${{ join(matrix.targets, '_') }} + path: | + ${{ steps.cargo-dist.outputs.paths }} + ${{ env.BUILD_MANIFEST_NAME }} + + # Build and package all the platform-agnostic(ish) things + build-global-artifacts: + needs: + - plan + - build-local-artifacts + runs-on: "ubuntu-22.04" + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + BUILD_MANIFEST_NAME: target/distrib/global-dist-manifest.json + steps: + - uses: actions/checkout@v7 + with: + persist-credentials: false + submodules: recursive + - name: Install cached dist + uses: actions/download-artifact@v8 + with: + name: cargo-dist-cache + path: ~/.cargo/bin/ + - run: chmod +x ~/.cargo/bin/dist + # Get all the local artifacts for the global tasks to use (for e.g. checksums) + - name: Fetch local artifacts + uses: actions/download-artifact@v8 + with: + pattern: artifacts-* + path: target/distrib/ + merge-multiple: true + - id: cargo-dist + shell: bash + run: | + dist build ${{ needs.plan.outputs.tag-flag }} --output-format=json "--artifacts=global" > dist-manifest.json + echo "dist ran successfully" + + # Parse out what we just built and upload it to scratch storage + echo "paths<> "$GITHUB_OUTPUT" + jq --raw-output ".upload_files[]" dist-manifest.json >> "$GITHUB_OUTPUT" + echo "EOF" >> "$GITHUB_OUTPUT" + + cp dist-manifest.json "$BUILD_MANIFEST_NAME" + - name: "Upload artifacts" + uses: actions/upload-artifact@v7 + with: + name: artifacts-build-global + path: | + ${{ steps.cargo-dist.outputs.paths }} + ${{ env.BUILD_MANIFEST_NAME }} + # Determines if we should publish/announce + host: + needs: + - plan + - build-local-artifacts + - build-global-artifacts + # Only run if we're "publishing", and only if plan, local and global didn't fail (skipped is fine) + if: ${{ always() && needs.plan.result == 'success' && needs.plan.outputs.publishing == 'true' && (needs.build-global-artifacts.result == 'skipped' || needs.build-global-artifacts.result == 'success') && (needs.build-local-artifacts.result == 'skipped' || needs.build-local-artifacts.result == 'success') }} + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + runs-on: "ubuntu-22.04" + outputs: + val: ${{ steps.host.outputs.manifest }} + steps: + - uses: actions/checkout@v7 + with: + persist-credentials: false + submodules: recursive + - name: Install cached dist + uses: actions/download-artifact@v8 + with: + name: cargo-dist-cache + path: ~/.cargo/bin/ + - run: chmod +x ~/.cargo/bin/dist + # Fetch artifacts from scratch-storage + - name: Fetch artifacts + uses: actions/download-artifact@v8 + with: + pattern: artifacts-* + path: target/distrib/ + merge-multiple: true + - id: host + shell: bash + run: | + dist host ${{ needs.plan.outputs.tag-flag }} --steps=upload --steps=release --output-format=json > dist-manifest.json + echo "artifacts uploaded and released successfully" + cat dist-manifest.json + echo "manifest=$(jq -c "." dist-manifest.json)" >> "$GITHUB_OUTPUT" + - name: "Upload dist-manifest.json" + uses: actions/upload-artifact@v7 + with: + # Overwrite the previous copy + name: artifacts-dist-manifest + path: dist-manifest.json + # Create a GitHub Release while uploading all files to it + - name: "Download GitHub Artifacts" + uses: actions/download-artifact@v8 + with: + pattern: artifacts-* + path: artifacts + merge-multiple: true + - name: Cleanup + run: | + # Remove the granular manifests + rm -f artifacts/*-dist-manifest.json + - name: Create GitHub Release + env: + PRERELEASE_FLAG: "${{ fromJson(steps.host.outputs.manifest).announcement_is_prerelease && '--prerelease' || '' }}" + ANNOUNCEMENT_TITLE: "${{ fromJson(steps.host.outputs.manifest).announcement_title }}" + ANNOUNCEMENT_BODY: "${{ fromJson(steps.host.outputs.manifest).announcement_github_body }}" + RELEASE_COMMIT: "${{ github.sha }}" + run: | + # Write and read notes from a file to avoid quoting breaking things + echo "$ANNOUNCEMENT_BODY" > $RUNNER_TEMP/notes.txt + + gh release create "${{ needs.plan.outputs.tag }}" --target "$RELEASE_COMMIT" $PRERELEASE_FLAG --title "$ANNOUNCEMENT_TITLE" --notes-file "$RUNNER_TEMP/notes.txt" artifacts/* + + announce: + needs: + - plan + - host + # use "always() && ..." to allow us to wait for all publish jobs while + # still allowing individual publish jobs to skip themselves (for prereleases). + # "host" however must run to completion, no skipping allowed! + if: ${{ always() && needs.host.result == 'success' }} + runs-on: "ubuntu-22.04" + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + steps: + - uses: actions/checkout@v7 + with: + persist-credentials: false + submodules: recursive diff --git a/userland/diffutils/.github/workflows/wasi.yml b/userland/diffutils/.github/workflows/wasi.yml new file mode 100644 index 0000000..3cb02cc --- /dev/null +++ b/userland/diffutils/.github/workflows/wasi.yml @@ -0,0 +1,28 @@ +# spell-checker:ignore wasip +name: WASI + +on: + pull_request: + push: + branches: + - main + +permissions: + contents: read + +# End the current execution if there is a new changeset in the PR. +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} + +jobs: + test_wasi: + name: Tests + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - uses: dtolnay/rust-toolchain@stable + with: + targets: wasm32-wasip1 + - name: check + run: cargo check --target wasm32-wasip1 diff --git a/userland/diffutils/.gitignore b/userland/diffutils/.gitignore new file mode 100644 index 0000000..515a64d --- /dev/null +++ b/userland/diffutils/.gitignore @@ -0,0 +1,3 @@ +/target +*.swp +/tests/test-results.json diff --git a/userland/diffutils/.pre-commit-config.yaml b/userland/diffutils/.pre-commit-config.yaml new file mode 100644 index 0000000..25bf13d --- /dev/null +++ b/userland/diffutils/.pre-commit-config.yaml @@ -0,0 +1,48 @@ +# See https://pre-commit.com for more information +# See https://pre-commit.com/hooks.html for more hooks +exclude: ^tests/fixtures/ +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v6.0.0 + hooks: + - id: check-added-large-files + - id: check-executables-have-shebangs + - id: check-json + exclude: '\.vscode/(cSpell|extensions)\.json' # cSpell.json and extensions.json use comments + - id: check-shebang-scripts-are-executable + exclude: '.+\.rs' # would be triggered by #![some_attribute] + - id: check-symlinks + - id: check-toml + - id: check-yaml + args: [ --allow-multiple-documents ] + - id: destroyed-symlinks + - id: end-of-file-fixer + - id: mixed-line-ending + args: [ --fix=lf ] + - id: trailing-whitespace + + - repo: local + hooks: + - id: rust-linting + name: Rust linting + description: Run cargo fmt on files included in the commit. + entry: cargo +stable fmt -- + pass_filenames: true + types: [file, rust] + language: system + - id: rust-clippy + name: Rust clippy + description: Run cargo clippy on files included in the commit. + entry: cargo +stable clippy --workspace --all-targets --all-features -- -D warnings + pass_filenames: false + types: [file, rust] + language: system + - id: cspell + name: Code spell checker (cspell) + description: Run cspell to check for spelling errors (if available). + entry: bash -c 'if command -v cspell >/dev/null 2>&1; then cspell --no-must-find-files -- "$@"; else echo "cspell not found, skipping spell check"; exit 0; fi' -- + pass_filenames: true + language: system + +ci: + skip: [rust-linting, rust-clippy, cspell] diff --git a/userland/diffutils/CONTRIBUTING.md b/userland/diffutils/CONTRIBUTING.md new file mode 100644 index 0000000..b2f59c2 --- /dev/null +++ b/userland/diffutils/CONTRIBUTING.md @@ -0,0 +1,32 @@ +# Contributing to diffutils + +Hi! Welcome to uutils/diffutils, and thanks for wanting to contribute! + +This project follows the shared conventions of the [uutils](https://github.com/uutils) +organization. Before opening a pull request, please read: + +- Our **[Review Guidelines](https://uutils.github.io/reviews/)** — what we expect + from a pull request and how reviews are carried out. +- Our community's [CODE_OF_CONDUCT.md](./CODE_OF_CONDUCT.md), if present. + +Finally, feel free to join our [Discord](https://discord.gg/wQVJbvJ)! + +> [!WARNING] +> uutils is original code and cannot contain any code from GNU or other +> strongly-licensed (GPL/LGPL) implementations. We **cannot** accept changes +> based on the GNU source code, and you **must not link** to it either. You may +> look at permissively-licensed implementations (MIT/BSD) and read the GNU +> *manuals* — never the GNU *source*. + +## In short + +- Discuss non-trivial changes in an issue **before** writing the code. +- Keep pull requests **small, self-contained, and descriptively titled** + (e.g. `diffutils: fix ...`). +- Make sure CI passes: tests are green, `rustfmt` is satisfied, and there are + no `clippy` warnings. +- Add tests for new behavior; don't let coverage regress. +- Write small, atomic commits annotated with the component you touched. + +See the [Review Guidelines](https://uutils.github.io/reviews/) for the full +details. diff --git a/userland/diffutils/COPYRIGHT b/userland/diffutils/COPYRIGHT new file mode 100644 index 0000000..22d70ff --- /dev/null +++ b/userland/diffutils/COPYRIGHT @@ -0,0 +1,8 @@ +Copyright (c) Michael Howell +Copyright (c) uutils developers + +Licensed under the Apache License, Version 2.0 or the MIT license +, at your +option. All files in the project carrying such notice may not be +copied, modified, or distributed except according to those terms. diff --git a/userland/diffutils/Cargo.lock b/userland/diffutils/Cargo.lock new file mode 100644 index 0000000..42d274c --- /dev/null +++ b/userland/diffutils/Cargo.lock @@ -0,0 +1,917 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "aho-corasick" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +dependencies = [ + "memchr", +] + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + +[[package]] +name = "anyhow" +version = "1.0.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" + +[[package]] +name = "approx" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cab112f0a86d568ea0e627cc1d6be74a1e9cd55214684db5561995f6dad897c6" +dependencies = [ + "num-traits", +] + +[[package]] +name = "assert_cmd" +version = "2.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2aa3a22042e45de04255c7bf3626e239f450200fd0493c1e382263544b20aea6" +dependencies = [ + "anstyle", + "bstr", + "libc", + "predicates", + "predicates-core", + "predicates-tree", + "wait-timeout", +] + +[[package]] +name = "autocfg" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" + +[[package]] +name = "bitflags" +version = "2.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843867be96c8daad0d758b57df9392b6d8d271134fce549de6ce169ff98a92af" + +[[package]] +name = "bstr" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63044e1ae8e69f3b5a92c736ca6269b8d12fa7efe39bf34ddb06d102cf0e2cab" +dependencies = [ + "memchr", + "regex-automata", + "serde", +] + +[[package]] +name = "bumpalo" +version = "3.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d20789868f4b01b2f2caec9f5c4e0213b41e3e5702a50157d699ae31ced2fcb" + +[[package]] +name = "cc" +version = "1.2.60" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43c5703da9466b66a946814e1adf53ea2c90f10063b86290cc9eb67ce3478a20" +dependencies = [ + "find-msvc-tools", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "cfg_aliases" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" + +[[package]] +name = "chacha20" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f8d983286843e49675a4b7a2d174efe136dc93a18d69130dd18198a6c167601" +dependencies = [ + "cfg-if", + "cpufeatures", + "rand_core", +] + +[[package]] +name = "chrono" +version = "0.4.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" +dependencies = [ + "iana-time-zone", + "js-sys", + "num-traits", + "wasm-bindgen", + "windows-link", +] + +[[package]] +name = "clap" +version = "4.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b193af5b67834b676abd72466a96c1024e6a6ad978a1f484bd90b85c94041351" +dependencies = [ + "clap_builder", +] + +[[package]] +name = "clap_builder" +version = "4.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f" +dependencies = [ + "anstyle", + "clap_lex", + "terminal_size", +] + +[[package]] +name = "clap_lex" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" + +[[package]] +name = "codspeed" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e7c21807b937b8e535f2745cd83c525fc250524021cd662b9ba4d3e399bb504" +dependencies = [ + "anyhow", + "cc", + "colored", + "getrandom", + "glob", + "nix", + "serde", + "serde_json", + "statrs", +] + +[[package]] +name = "codspeed-divan-compat" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc7e48c4e555e60b894144b14df1e893a7317b9f4000b1492ace3041479b12cc" +dependencies = [ + "clap", + "codspeed", + "codspeed-divan-compat-macros", + "codspeed-divan-compat-walltime", + "regex", +] + +[[package]] +name = "codspeed-divan-compat-macros" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1eb62a7a8d4cc6b79b18dcbac670b3202f4c8c901ca8ea701ab42190f0b14799" +dependencies = [ + "divan-macros", + "itertools", + "proc-macro-crate", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "codspeed-divan-compat-walltime" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15c4e77a97571b6214b976cee7e2a564215d0a5bee9520a3cdab86106d0b31a3" +dependencies = [ + "cfg-if", + "clap", + "codspeed", + "condtype", + "divan-macros", + "libc", + "regex-lite", +] + +[[package]] +name = "colored" +version = "3.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "faf9468729b8cbcea668e36183cb69d317348c2e08e994829fb56ebfdfbaac34" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "condtype" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf0a07a401f374238ab8e2f11a104d2851bf9ce711ec69804834de8af45c7af" + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "cpufeatures" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" +dependencies = [ + "libc", +] + +[[package]] +name = "diff" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56254986775e3233ffa9c4d7d3faaf6d36a2c09d30b20687e9f88bc8bafc16c8" + +[[package]] +name = "difflib" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6184e33543162437515c2e2b48714794e37845ec9851711914eec9d308f6ebe8" + +[[package]] +name = "diffutils" +version = "0.6.0" +dependencies = [ + "assert_cmd", + "chrono", + "codspeed-divan-compat", + "diff", + "itoa", + "predicates", + "pretty_assertions", + "rand", + "regex", + "same-file", + "tempfile", + "unicode-width", +] + +[[package]] +name = "divan-macros" +version = "0.1.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8dc51d98e636f5e3b0759a39257458b22619cac7e96d932da6eeb052891bb67c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "either" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "fastrand" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6" + +[[package]] +name = "find-msvc-tools" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" + +[[package]] +name = "float-cmp" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b09cf3155332e944990140d967ff5eceb70df778b34f77d8075db46e4704e6d8" +dependencies = [ + "num-traits", +] + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "libc", + "r-efi", + "rand_core", +] + +[[package]] +name = "glob" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" + +[[package]] +name = "hashbrown" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f467dd6dccf739c208452f8014c75c18bb8301b050ad1cfb27153803edb0f51" + +[[package]] +name = "iana-time-zone" +version = "0.1.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "indexmap" +version = "2.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +dependencies = [ + "equivalent", + "hashbrown", +] + +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "js-sys" +version = "0.3.94" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e04e2ef80ce82e13552136fabeef8a5ed1f985a96805761cbb9a2c34e7664d9" +dependencies = [ + "once_cell", + "wasm-bindgen", +] + +[[package]] +name = "libc" +version = "0.2.184" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48f5d2a454e16a5ea0f4ced81bd44e4cfc7bd3a507b61887c99fd3538b28e4af" + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "log" +version = "0.4.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" + +[[package]] +name = "memchr" +version = "2.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" + +[[package]] +name = "nix" +version = "0.31.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d6d0705320c1e6ba1d912b5e37cf18071b6c2e9b7fa8215a1e8a7651966f5d3" +dependencies = [ + "bitflags", + "cfg-if", + "cfg_aliases", + "libc", +] + +[[package]] +name = "normalize-line-endings" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61807f77802ff30975e01f4f071c8ba10c022052f98b3294119f3e615d13e5be" + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "predicates" +version = "3.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ada8f2932f28a27ee7b70dd6c1c39ea0675c55a36879ab92f3a715eaa1e63cfe" +dependencies = [ + "anstyle", + "difflib", + "float-cmp", + "normalize-line-endings", + "predicates-core", + "regex", +] + +[[package]] +name = "predicates-core" +version = "1.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cad38746f3166b4031b1a0d39ad9f954dd291e7854fcc0eed52ee41a0b50d144" + +[[package]] +name = "predicates-tree" +version = "1.0.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0de1b847b39c8131db0467e9df1ff60e6d0562ab8e9a16e568ad0fdb372e2f2" +dependencies = [ + "predicates-core", + "termtree", +] + +[[package]] +name = "pretty_assertions" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ae130e2f271fbc2ac3a40fb1d07180839cdbbe443c7a27e1e3c13c5cac0116d" +dependencies = [ + "diff", + "yansi", +] + +[[package]] +name = "proc-macro-crate" +version = "3.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e67ba7e9b2b56446f1d419b1d807906278ffa1a658a8a5d8a39dcb1f5a78614f" +dependencies = [ + "toml_edit", +] + +[[package]] +name = "proc-macro2" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "rand" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65c9fb96cbc91e3478eaae79a69fcd3f1ae4ad052e471fe6732fff548984b4af" +dependencies = [ + "chacha20", + "getrandom", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c8d0fd677905edcbeedbf2edb6494d676f0e98d54d5cf9bda0b061cb8fb8aba" + +[[package]] +name = "regex" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fcfdb36bda0c880c5931cdc7a2bcdc8ba4556847b9d912bca70bc94708711ad" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-lite" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cab834c73d247e67f4fae452806d17d3c7501756d98c8808d7c9c7aa7d18f973" + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "rustix" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys", +] + +[[package]] +name = "rustversion" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.149" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + +[[package]] +name = "statrs" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a3fe7c28c6512e766b0874335db33c94ad7b8f9054228ae1c2abd47ce7d335e" +dependencies = [ + "approx", + "num-traits", +] + +[[package]] +name = "syn" +version = "2.0.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom", + "once_cell", + "rustix", + "windows-sys", +] + +[[package]] +name = "terminal_size" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "230a1b821ccbd75b185820a1f1ff7b14d21da1e442e22c0863ea5f08771a8874" +dependencies = [ + "rustix", + "windows-sys", +] + +[[package]] +name = "termtree" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f50febec83f5ee1df3015341d8bd429f2d1cc62bcba7ea2076759d315084683" + +[[package]] +name = "toml_datetime" +version = "1.1.1+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3165f65f62e28e0115a00b2ebdd37eb6f3b641855f9d636d3cd4103767159ad7" +dependencies = [ + "serde_core", +] + +[[package]] +name = "toml_edit" +version = "0.25.11+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b59c4d22ed448339746c59b905d24568fcbb3ab65a500494f7b8c3e97739f2b" +dependencies = [ + "indexmap", + "toml_datetime", + "toml_parser", + "winnow", +] + +[[package]] +name = "toml_parser" +version = "1.1.2+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2abe9b86193656635d2411dc43050282ca48aa31c2451210f4202550afb7526" +dependencies = [ + "winnow", +] + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "unicode-width" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" + +[[package]] +name = "wait-timeout" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ac3b126d3914f9849036f826e054cbabdc8519970b8998ddaf3b5bd3c65f11" +dependencies = [ + "libc", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0551fc1bb415591e3372d0bc4780db7e587d84e2a7e79da121051c5c4b89d0b0" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fbdf9a35adf44786aecd5ff89b4563a90325f9da0923236f6104e603c7e86be" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dca9693ef2bab6d4e6707234500350d8dad079eb508dca05530c85dc3a529ff2" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39129a682a6d2d841b6c429d0c51e5cb0ed1a03829d8b3d1e69a011e62cb3d3b" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "winnow" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09dac053f1cd375980747450bfc7250c264eaae0583872e845c0c7cd578872b5" +dependencies = [ + "memchr", +] + +[[package]] +name = "yansi" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfe53a6657fd280eaa890a3bc59152892ffa3e30101319d168b781ed6529b049" + +[[package]] +name = "zmij" +version = "1.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" diff --git a/userland/diffutils/Cargo.toml b/userland/diffutils/Cargo.toml new file mode 100644 index 0000000..f788ab3 --- /dev/null +++ b/userland/diffutils/Cargo.toml @@ -0,0 +1,91 @@ +# THIS FILE IS AUTOMATICALLY GENERATED BY CARGO +# +# When uploading crates to the registry Cargo will automatically +# "normalize" Cargo.toml files for maximal compatibility +# with all versions of Cargo and also rewrite `path` dependencies +# to registry (e.g., crates.io) dependencies. +# +# If you are reading this file be aware that the original Cargo.toml +# will likely look very different (and much more reasonable). +# See Cargo.toml.orig for the original contents. + +[package] +edition = "2021" +name = "diffutils" +version = "0.6.0" +build = false +autolib = false +autobins = false +autoexamples = false +autotests = false +autobenches = false +description = "A CLI app for generating diff files" +readme = "README.md" +license = "MIT OR Apache-2.0" +repository = "https://github.com/uutils/diffutils" + +[features] +feat_bench_not_cmp = [] +feat_bench_not_diff = [] + +[lib] +name = "diffutilslib" +path = "src/lib.rs" + +[[bin]] +name = "diffutils" +path = "src/main.rs" + +[[test]] +name = "integration" +path = "tests/integration.rs" + +[[bench]] +name = "bench_diffutils" +path = "benches/bench-diffutils.rs" +harness = false + +[dependencies.chrono] +version = "0.4.38" + +[dependencies.diff] +version = "0.1.13" + +[dependencies.itoa] +version = "1.0.11" + +[dependencies.regex] +version = "1.10.4" + +[dependencies.same-file] +version = "1.0.6" + +[dependencies.unicode-width] +version = "0.2.0" + +[dev-dependencies.assert_cmd] +version = "2.0.14" + +[dev-dependencies.divan] +version = "6.0.0" +package = "codspeed-divan-compat" + +[dev-dependencies.predicates] +version = "3.1.0" + +[dev-dependencies.pretty_assertions] +version = "1.4.0" + +[dev-dependencies.rand] +version = "0.10.0" + +[dev-dependencies.tempfile] +version = "3.26.0" + +[profile.dist] +inherits = "release" + +[profile.release] +lto = "thin" +codegen-units = 1 +panic = "abort" diff --git a/userland/diffutils/Cargo.toml.orig b/userland/diffutils/Cargo.toml.orig new file mode 100644 index 0000000..719a311 --- /dev/null +++ b/userland/diffutils/Cargo.toml.orig @@ -0,0 +1,52 @@ +[package] +name = "diffutils" +version = "0.6.0" +edition = "2021" +description = "A CLI app for generating diff files" +license = "MIT OR Apache-2.0" +repository = "https://github.com/uutils/diffutils" + +[lib] +name = "diffutilslib" +path = "src/lib.rs" + +[[bin]] +name = "diffutils" +path = "src/main.rs" + +[dependencies] +chrono = "0.4.38" +diff = "0.1.13" +itoa = "1.0.11" +regex = "1.10.4" +same-file = "1.0.6" +unicode-width = "0.2.0" + +[dev-dependencies] +assert_cmd = "2.0.14" +divan = { version = "6.0.0", package = "codspeed-divan-compat" } +pretty_assertions = "1.4.0" +predicates = "3.1.0" +rand = "0.10.0" +tempfile = "3.26.0" + +[profile.release] +lto = "thin" +codegen-units = 1 +panic = "abort" + +# alias profile for 'dist' +[profile.dist] +inherits = "release" + +[[bench]] +name = "bench_diffutils" +path = "benches/bench-diffutils.rs" +harness = false + +[features] +# default = ["feat_bench_not_diff"] +# Turn bench for diffutils cmp off +feat_bench_not_cmp = [] +# Turn bench for diffutils diff off +feat_bench_not_diff = [] diff --git a/userland/diffutils/LICENSE-APACHE b/userland/diffutils/LICENSE-APACHE new file mode 100644 index 0000000..1b5ec8b --- /dev/null +++ b/userland/diffutils/LICENSE-APACHE @@ -0,0 +1,176 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS diff --git a/userland/diffutils/LICENSE-MIT b/userland/diffutils/LICENSE-MIT new file mode 100644 index 0000000..31aa793 --- /dev/null +++ b/userland/diffutils/LICENSE-MIT @@ -0,0 +1,23 @@ +Permission is hereby granted, free of charge, to any +person obtaining a copy of this software and associated +documentation files (the "Software"), to deal in the +Software without restriction, including without +limitation the rights to use, copy, modify, merge, +publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software +is furnished to do so, subject to the following +conditions: + +The above copyright notice and this permission notice +shall be included in all copies or substantial portions +of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF +ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED +TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A +PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. diff --git a/userland/diffutils/PATCHES.md b/userland/diffutils/PATCHES.md new file mode 100644 index 0000000..0bf8c37 --- /dev/null +++ b/userland/diffutils/PATCHES.md @@ -0,0 +1,14 @@ +# Recursive directory comparison + +Source: crates.io `diffutils` 0.6.0, https://github.com/uutils/diffutils. +The vendor copy retains the upstream licenses. + +The public library exposes comparison functions but not the diff CLI entry point. +The vendor copy retains that CLI parser and dispatcher. + +- `src/params.rs` accepts `-r` and `--recursive`. +- `src/diff.rs` recursively compares directory entries in lexical order. +- The recursive path reuses the upstream file comparison implementation. +- `tools.txt` exposes the component as `diff` and `cmp`. + +`search-test.nu` compares identical directories and changed nested files against GNU diff. diff --git a/userland/diffutils/README.md b/userland/diffutils/README.md new file mode 100644 index 0000000..489e5de --- /dev/null +++ b/userland/diffutils/README.md @@ -0,0 +1,61 @@ +[![Crates.io](https://img.shields.io/crates/v/diffutils.svg)](https://crates.io/crates/diffutils) +[![Discord](https://img.shields.io/badge/discord-join-7289DA.svg?logo=discord&longCache=true&style=flat)](https://discord.gg/wQVJbvJ) +[![License](http://img.shields.io/badge/license-MIT-blue.svg)](https://github.com/uutils/diffutils/blob/main/LICENSE) +[![dependency status](https://deps.rs/repo/github/uutils/diffutils/status.svg)](https://deps.rs/repo/github/uutils/diffutils) +[![CodSpeed](https://img.shields.io/endpoint?url=https://codspeed.io/badge.json)](https://codspeed.io/uutils/diffutils?utm_source=badge) + +[![CodeCov](https://codecov.io/gh/uutils/diffutils/branch/main/graph/badge.svg)](https://codecov.io/gh/uutils/diffutils) + +The goal of this package is to be a drop-in replacement for the [diffutils commands](https://www.gnu.org/software/diffutils/) (diff, cmp, diff3, sdiff) in Rust. + +Based on the incomplete diff generator in https://github.com/rust-lang/rust/blob/master/src/tools/compiletest/src/runtest.rs, and made to be compatible with GNU's diff and patch tools. + + +## Installation + +Ensure you have Rust installed on your system. You can install Rust through [rustup](https://rustup.rs/). + +Clone the repository and build the project using Cargo: + +```bash +git clone https://github.com/uutils/diffutils.git +cd diffutils +cargo build --release +``` + +## Example + +```bash + +cat <fruits_old.txt +Apple +Banana +Cherry +EOF + +cat <fruits_new.txt +Apple +Fig +Cherry +EOF + +$ cargo run -- diff -u fruits_old.txt fruits_new.txt + Finished dev [unoptimized + debuginfo] target(s) in 0.00s + Running `target/debug/diffutils -u fruits_old.txt fruits_new.txt` +--- fruits_old.txt ++++ fruits_new.txt +@@ -1,3 +1,3 @@ + Apple +-Banana ++Fig + Cherry + +``` + +## License + +This project is distributed under the terms of both the MIT license and the +Apache License (Version 2.0). + +See [LICENSE-APACHE](LICENSE-APACHE), [LICENSE-MIT](LICENSE-MIT), and +[COPYRIGHT](COPYRIGHT) for details. diff --git a/userland/diffutils/SECURITY.md b/userland/diffutils/SECURITY.md new file mode 100644 index 0000000..7a79a3b --- /dev/null +++ b/userland/diffutils/SECURITY.md @@ -0,0 +1,44 @@ +# Security Policy + +## Supported Versions + +We provide security updates only for the latest released version of `uutils/diffutils`. +Older versions may not receive patches. +If you are using a version packaged by your Linux distribution, please check with your distribution maintainers for their update policy. + +--- + +## Reporting a Vulnerability + +**Do not open public GitHub issues for security vulnerabilities.** +This prevents accidental disclosure before a fix is available. + +Instead, please use the following method: + +- **Email:** [sylvestre@debian.org](mailto:Sylvestre@debian.org) +- **Encryption (optional):** You may encrypt your report using our PGP key: +Fingerprint: B60D B599 4D39 BEC4 D1A9 5CCF 7E65 28DA 752F 1BE1 +--- + +### What to Include in Your Report + +To help us investigate and resolve the issue quickly, please include as much detail as possible: + +- **Type of issue:** e.g. privilege escalation, information disclosure. +- **Location in the source:** file path, commit hash, branch, or tag. +- **Steps to reproduce:** exact commands, test cases, or scripts. +- **Special configuration:** any flags, environment variables, or system setup required. +- **Affected systems:** OS/distribution and version(s) where the issue occurs. +- **Impact:** your assessment of the potential severity (DoS, RCE, data leak, etc.). + +--- + +## Disclosure Policy + +We follow a **Coordinated Vulnerability Disclosure (CVD)** process: + +1. We will acknowledge receipt of your report within **10 days**. +2. We will investigate, reproduce, and assess the issue. +3. We will provide a timeline for developing and releasing a fix. +4. Once a fix is available, we will publish a GitHub Security Advisory. +5. You will be credited in the advisory unless you request anonymity. diff --git a/userland/diffutils/benches/bench-diffutils.rs b/userland/diffutils/benches/bench-diffutils.rs new file mode 100644 index 0000000..e506b3f --- /dev/null +++ b/userland/diffutils/benches/bench-diffutils.rs @@ -0,0 +1,377 @@ +// This file is part of the uutils diffutils package. +// +// For the full copyright and license information, please view the LICENSE-* +// files that was distributed with this source code. + +//! Benches for all utils in diffutils. +//! +//! There is a file generator included to create files of different sizes for comparison. \ +//! Set the TEMP_DIR const to keep the files. df_to_ files have small changes in them, search for '#'. \ +//! File generation up to 1 GB is really fast, Benchmarking above 100 MB takes very long. + +/// Generate test files with these sizes in KB. +const FILE_SIZE_KILO_BYTES: [u64; 4] = [100, 1 * MB, 10 * MB, 25 * MB]; +// const FILE_SIZE_KILO_BYTES: [u64; 3] = [100, 1 * MB, 5 * MB]; +// Empty String to use TempDir (files will be removed after test) or specify dir to keep generated files +const TEMP_DIR: &str = ""; +const NUM_DIFF: u64 = 4; +// just for FILE_SIZE_KILO_BYTES +const MB: u64 = 1_000; +const CHANGE_CHAR: u8 = b'#'; + +#[cfg(not(feature = "feat_bench_not_cmp"))] +mod diffutils_cmp { + use std::hint::black_box; + + use diffutilslib::cmp; + use divan::Bencher; + + use crate::{binary, prepare::*, FILE_SIZE_KILO_BYTES}; + + #[divan::bench(args = FILE_SIZE_KILO_BYTES)] + fn cmp_compare_files_equal(bencher: Bencher, kb: u64) { + let (from, to) = get_context().get_test_files_equal(kb); + let cmd = format!("cmp {from} {to}"); + let opts = str_to_options(&cmd).into_iter().peekable(); + let params = cmp::parse_params(opts).unwrap(); + + bencher + // .with_inputs(|| prepare::cmp_params_identical_testfiles(lines)) + .with_inputs(|| params.clone()) + .bench_refs(|params| black_box(cmp::cmp(¶ms).unwrap())); + } + + // bench the actual compare; cmp exits on first difference + #[divan::bench(args = FILE_SIZE_KILO_BYTES)] + fn cmp_compare_files_different(bencher: Bencher, bytes: u64) { + let (from, to) = get_context().get_test_files_different(bytes); + let cmd = format!("cmp {from} {to} -s"); + let opts = str_to_options(&cmd).into_iter().peekable(); + let params = cmp::parse_params(opts).unwrap(); + + bencher + // .with_inputs(|| prepare::cmp_params_identical_testfiles(lines)) + .with_inputs(|| params.clone()) + .bench_refs(|params| black_box(cmp::cmp(¶ms).unwrap())); + } + + // bench original GNU cmp + #[divan::bench(args = FILE_SIZE_KILO_BYTES)] + fn cmd_cmp_gnu_equal(bencher: Bencher, bytes: u64) { + let (from, to) = get_context().get_test_files_equal(bytes); + let args_str = format!("{from} {to}"); + bencher + // .with_inputs(|| prepare::cmp_params_identical_testfiles(lines)) + .with_inputs(|| args_str.clone()) + .bench_refs(|cmd_args| binary::bench_binary("cmp", cmd_args)); + } + + // bench the compiled release version + #[divan::bench(args = FILE_SIZE_KILO_BYTES)] + fn cmd_cmp_release_equal(bencher: Bencher, bytes: u64) { + let (from, to) = get_context().get_test_files_equal(bytes); + let args_str = format!("cmp {from} {to}"); + + bencher + // .with_inputs(|| prepare::cmp_params_identical_testfiles(lines)) + .with_inputs(|| args_str.clone()) + .bench_refs(|cmd_args| binary::bench_binary("target/release/diffutils", cmd_args)); + } +} + +#[cfg(not(feature = "feat_bench_not_diff"))] +mod diffutils_diff { + // use std::hint::black_box; + + use crate::{binary, prepare::*, FILE_SIZE_KILO_BYTES}; + // use diffutilslib::params; + use divan::Bencher; + + // bench the actual compare + // TODO diff does not have a diff function + // #[divan::bench(args = [100_000,10_000])] + // fn diff_compare_files(bencher: Bencher, bytes: u64) { + // let (from, to) = gen_testfiles(lines, 0, "id"); + // let cmd = format!("cmp {from} {to}"); + // let opts = str_to_options(&cmd).into_iter().peekable(); + // let params = params::parse_params(opts).unwrap(); + // + // bencher + // // .with_inputs(|| prepare::cmp_params_identical_testfiles(lines)) + // .with_inputs(|| params.clone()) + // .bench_refs(|params| diff::diff(¶ms).unwrap()); + // } + + // bench original GNU diff + #[divan::bench(args = FILE_SIZE_KILO_BYTES)] + fn cmd_diff_gnu_equal(bencher: Bencher, bytes: u64) { + let (from, to) = get_context().get_test_files_equal(bytes); + let args_str = format!("{from} {to}"); + bencher + // .with_inputs(|| prepare::cmp_params_identical_testfiles(lines)) + .with_inputs(|| args_str.clone()) + .bench_refs(|cmd_args| binary::bench_binary("diff", cmd_args)); + } + + // bench the compiled release version + #[divan::bench(args = FILE_SIZE_KILO_BYTES)] + fn cmd_diff_release_equal(bencher: Bencher, bytes: u64) { + let (from, to) = get_context().get_test_files_equal(bytes); + let args_str = format!("diff {from} {to}"); + + bencher + // .with_inputs(|| prepare::cmp_params_identical_testfiles(lines)) + .with_inputs(|| args_str.clone()) + .bench_refs(|cmd_args| binary::bench_binary("target/release/diffutils", cmd_args)); + } +} + +mod parser { + use std::hint::black_box; + + use diffutilslib::{cmp, params}; + use divan::Bencher; + + use crate::prepare::str_to_options; + + // bench the time it takes to parse the command line arguments + #[divan::bench] + fn cmp_parser(bencher: Bencher) { + let cmd = "cmd file_1.txt file_2.txt -bl n10M --ignore-initial=100KiB:1MiB"; + let args = str_to_options(&cmd).into_iter().peekable(); + bencher + .with_inputs(|| args.clone()) + .bench_values(|data| black_box(cmp::parse_params(data))); + } + + // // test the impact on the benchmark if not converting the cmd to Vec (doubles for parse) + // #[divan::bench] + // fn cmp_parser_no_prepare() { + // let cmd = "cmd file_1.txt file_2.txt -bl n10M --ignore-initial=100KiB:1MiB"; + // let args = str_to_options(&cmd).into_iter().peekable(); + // let _ = cmp::parse_params(args); + // } + + // bench the time it takes to parse the command line arguments + #[divan::bench] + fn diff_parser(bencher: Bencher) { + let cmd = "diff file_1.txt file_2.txt -s --brief --expand-tabs --width=100"; + let args = str_to_options(&cmd).into_iter().peekable(); + bencher + .with_inputs(|| args.clone()) + .bench_values(|data| black_box(params::parse_params(data))); + } +} + +mod prepare { + use std::{ + ffi::OsString, + fs::{self, File}, + io::{BufWriter, Write}, + path::Path, + sync::OnceLock, + }; + + use rand::RngExt; + use tempfile::TempDir; + + use crate::{CHANGE_CHAR, FILE_SIZE_KILO_BYTES, NUM_DIFF, TEMP_DIR}; + + // file lines and .txt will be added + const FROM_FILE: &str = "from_file"; + const TO_FILE: &str = "to_file"; + const LINE_LENGTH: usize = 60; + + /// Contains test data (file names) which only needs to be created once. + #[derive(Debug, Default)] + pub struct BenchContext { + pub tmp_dir: Option, + pub dir: String, + pub files_equal: Vec<(String, String)>, + pub files_different: Vec<(String, String)>, + } + + impl BenchContext { + pub fn get_path(&self) -> &Path { + match &self.tmp_dir { + Some(tmp) => tmp.path(), + None => Path::new(&self.dir), + } + } + + pub fn get_test_files_equal(&self, kb: u64) -> &(String, String) { + let p = FILE_SIZE_KILO_BYTES.iter().position(|f| *f == kb).unwrap(); + &self.files_equal[p] + } + + #[allow(unused)] + pub fn get_test_files_different(&self, kb: u64) -> &(String, String) { + let p = FILE_SIZE_KILO_BYTES.iter().position(|f| *f == kb).unwrap(); + &self.files_different[p] + } + } + + // Since each bench function is separate in Divan it is more difficult to dynamically create test data. + // This keeps the TempDir alive until the program exits and generates the files only once. + static SHARED_CONTEXT: OnceLock = OnceLock::new(); + /// Creates the test files once and provides them to all tests. + pub fn get_context() -> &'static BenchContext { + SHARED_CONTEXT.get_or_init(|| { + let mut ctx = BenchContext::default(); + if TEMP_DIR.is_empty() { + let tmp_dir = TempDir::new().expect("Failed to create temp dir"); + ctx.tmp_dir = Some(tmp_dir); + } else { + // uses current directory, the generated files are kept + let path = Path::new(TEMP_DIR); + if !path.exists() { + fs::create_dir_all(path).expect("Path {path} could not be created"); + } + ctx.dir = TEMP_DIR.to_string(); + }; + + // generate test bytes + for kb in FILE_SIZE_KILO_BYTES { + let f = generate_test_files_bytes(ctx.get_path(), kb * 1000, 0, "eq") + .expect("generate_test_files failed"); + ctx.files_equal.push(f); + let f = generate_test_files_bytes(ctx.get_path(), kb * 1000, NUM_DIFF, "df") + .expect("generate_test_files failed"); + ctx.files_different.push(f); + } + + ctx + }) + } + + pub fn str_to_options(opt: &str) -> Vec { + let s: Vec = opt + .split(" ") + .into_iter() + .filter(|s| !s.is_empty()) + .map(|s| OsString::from(s)) + .collect(); + + s + } + + /// Generates two test files for comparison with size. + /// + /// Each line consists of 10 words with 5 letters, giving a line length of 60 bytes. + /// If num_differences is set, '#' will be inserted between the first two words of a line, + /// evenly spaced in the file. 1 will add the change in the last line, so the comparison takes longest. + fn generate_test_files_bytes( + dir: &Path, + bytes: u64, + num_differences: u64, + id: &str, + ) -> std::io::Result<(String, String)> { + let id = if id.is_empty() { + "".to_string() + } else { + format!("{id}_") + }; + let f1 = format!("{id}{FROM_FILE}_{bytes}.txt"); + let f2 = format!("{id}{TO_FILE}_{bytes}.txt"); + let from_path = dir.join(f1); + let to_path = dir.join(f2); + + generate_file_bytes(&from_path, &to_path, bytes, num_differences)?; + + Ok(( + from_path.to_string_lossy().to_string(), + to_path.to_string_lossy().to_string(), + )) + } + + fn generate_file_bytes( + from_name: &Path, + to_name: &Path, + bytes: u64, + num_differences: u64, + ) -> std::io::Result<()> { + let file_from = File::create(from_name)?; + let file_to = File::create(to_name)?; + // for int division, lines will be smaller than requested bytes + let n_lines = bytes / LINE_LENGTH as u64; + let change_every_n_lines = if num_differences == 0 { + 0 + } else { + let c = n_lines / num_differences; + if c == 0 { + 1 + } else { + c + } + }; + // Use a larger 128KB buffer for massive files + let mut writer_from = BufWriter::with_capacity(128 * 1024, file_from); + let mut writer_to = BufWriter::with_capacity(128 * 1024, file_to); + let mut rng = rand::rng(); + + // Each line: (5 chars * 10 words) + 9 spaces + 1 newline = 60 bytes + let mut line_buffer = [b' '; 60]; + line_buffer[59] = b'\n'; // Set the newline once at the end + + for i in (0..n_lines).rev() { + // Fill only the letter positions, skipping spaces and the newline + for word_idx in 0..10 { + let start = word_idx * 6; // Each word + space block is 6 bytes + for i in 0..5 { + line_buffer[start + i] = rng.random_range(b'a'..b'z' + 1); + } + } + + // Write the raw bytes directly to both files + writer_from.write_all(&line_buffer)?; + // make changes in the file + if num_differences == 0 { + writer_to.write_all(&line_buffer)?; + } else { + if i % change_every_n_lines == 0 && n_lines - i > 2 { + line_buffer[5] = CHANGE_CHAR; + } + writer_to.write_all(&line_buffer)?; + line_buffer[5] = b' '; + } + } + + // create last line + let missing = (bytes - n_lines as u64 * LINE_LENGTH as u64) as usize; + if missing > 0 { + for word_idx in 0..10 { + let start = word_idx * 6; // Each word + space block is 6 bytes + for i in 0..5 { + line_buffer[start + i] = rng.random_range(b'a'..b'z' + 1); + } + } + line_buffer[missing - 1] = b'\n'; + writer_from.write_all(&line_buffer[0..missing])?; + writer_to.write_all(&line_buffer[0..missing])?; + } + + writer_from.flush()?; + writer_to.flush()?; + + Ok(()) + } +} + +mod binary { + use std::process::Command; + + use crate::prepare::str_to_options; + + pub fn bench_binary(program: &str, cmd_args: &str) -> std::process::ExitStatus { + let args = str_to_options(cmd_args); + Command::new(program) + .args(args) + .status() + .expect("Failed to execute binary") + } +} + +fn main() { + // Run registered benchmarks. + divan::main(); +} diff --git a/userland/diffutils/dist-workspace.toml b/userland/diffutils/dist-workspace.toml new file mode 100644 index 0000000..3cdcaef --- /dev/null +++ b/userland/diffutils/dist-workspace.toml @@ -0,0 +1,15 @@ +[workspace] +members = ["cargo:."] + +# Config for 'dist' +[dist] +# The preferred dist version to use in CI (Cargo.toml SemVer syntax) +cargo-dist-version = "0.33.0" +# CI backends to support +ci = "github" +# The installers to generate for each app +installers = [] +# Target platforms to build apps for (Rust target-triple syntax) +targets = ["aarch64-apple-darwin", "aarch64-unknown-linux-gnu", "x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc"] +# Skip checking whether the specified configuration files are up to date +allow-dirty = ["ci"] diff --git a/userland/diffutils/renovate.json b/userland/diffutils/renovate.json new file mode 100644 index 0000000..5db72dd --- /dev/null +++ b/userland/diffutils/renovate.json @@ -0,0 +1,6 @@ +{ + "$schema": "https://docs.renovatebot.com/renovate-schema.json", + "extends": [ + "config:recommended" + ] +} diff --git a/userland/diffutils/src/cmp.rs b/userland/diffutils/src/cmp.rs new file mode 100644 index 0000000..0f7250c --- /dev/null +++ b/userland/diffutils/src/cmp.rs @@ -0,0 +1,1203 @@ +// This file is part of the uutils diffutils package. +// +// For the full copyright and license information, please view the LICENSE-* +// files that was distributed with this source code. + +use crate::utils::format_failure_to_read_input_file; +use std::env::{self, ArgsOs}; +use std::ffi::OsString; +use std::io::{BufRead, BufReader, BufWriter, Read, Write}; +use std::iter::Peekable; +use std::process::ExitCode; +use std::{cmp, fs, io}; + +#[cfg(unix)] +use std::os::fd::{AsRawFd, FromRawFd}; + +#[cfg(unix)] +use std::os::unix::fs::MetadataExt; + +#[cfg(target_os = "windows")] +use std::os::windows::fs::MetadataExt; + +/// for --bytes, so really large number limits can be expressed, like 1Y. +pub type BytesLimitU64 = u64; +// ignore initial is currently limited to u64, as take(skip) is used. +pub type SkipU64 = u64; + +#[derive(Clone, Debug, Default, Eq, PartialEq)] +pub struct Params { + executable: OsString, + from: OsString, + to: OsString, + print_bytes: bool, + skip_a: Option, + skip_b: Option, + max_bytes: Option, + verbose: bool, + quiet: bool, +} + +#[inline] +fn usage_string(executable: &str) -> String { + format!("Usage: {executable} ") +} + +#[cfg(unix)] +fn is_stdout_dev_null() -> bool { + let Ok(dev_null) = fs::metadata("/dev/null") else { + return false; + }; + + let stdout_fd = io::stdout().lock().as_raw_fd(); + + // SAFETY: we have exclusive access to stdout right now. + let stdout_file = unsafe { fs::File::from_raw_fd(stdout_fd) }; + let Ok(stdout) = stdout_file.metadata() else { + return false; + }; + + let is_dev_null = stdout.dev() == dev_null.dev() && stdout.ino() == dev_null.ino(); + + // Don't let File close the fd. It's unfortunate that File doesn't have a leak_fd(). + std::mem::forget(stdout_file); + + is_dev_null +} + +#[cfg(not(any(unix, target_os = "windows")))] +fn is_stdout_dev_null() -> bool { + false +} + +pub fn parse_params>(mut opts: Peekable) -> Result { + let executable = opts.next().ok_or("Usage: ".to_string())?; + let executable_str = executable.to_string_lossy().to_string(); + + let parse_skip = |param: &str, skip_desc: &str| -> Result { + let suffix_start = param + .find(|b: char| !b.is_ascii_digit()) + .unwrap_or(param.len()); + let mut num = match param[..suffix_start].parse::() { + Ok(num) => num, + Err(e) if *e.kind() == std::num::IntErrorKind::PosOverflow => SkipU64::MAX, + Err(_) => { + return Err(format!( + "{executable_str}: invalid --ignore-initial value '{skip_desc}'" + )) + } + }; + + if suffix_start != param.len() { + // Note that GNU cmp advertises supporting up to Y, but fails if you try + // to actually use anything beyond E. + let multiplier: SkipU64 = match ¶m[suffix_start..] { + "kB" => 1_000, + "K" => 1_024, + "MB" => 1_000_000, + "M" => 1_048_576, + "GB" => 1_000_000_000, + "G" => 1_073_741_824, + "TB" => 1_000_000_000_000, + "T" => 1_099_511_627_776, + "PB" => 1_000_000_000_000_000, + "P" => 1_125_899_906_842_624, + "EB" => 1_000_000_000_000_000_000, + "E" => 1_152_921_504_606_846_976, + // TODO setting usize:MAX does not mimic GNU cmp behavior, it should be an error. + "ZB" => SkipU64::MAX, // 1_000_000_000_000_000_000_000, + "Z" => SkipU64::MAX, // 1_180_591_620_717_411_303_424, + "YB" => SkipU64::MAX, // 1_000_000_000_000_000_000_000_000, + "Y" => SkipU64::MAX, // 1_208_925_819_614_629_174_706_176, + _ => { + return Err(format!( + "{executable_str}: invalid --ignore-initial value '{skip_desc}'" + )); + } + }; + + num = match num.overflowing_mul(multiplier) { + (n, false) => n, + _ => SkipU64::MAX, + } + } + + Ok(num) + }; + + let mut params = Params { + executable, + ..Default::default() + }; + let mut from = None; + let mut to = None; + let mut skip_pos1 = None; + let mut skip_pos2 = None; + while let Some(param) = opts.next() { + if param == "--" { + break; + } + if param == "-" { + if from.is_none() { + from = Some(param); + } else if to.is_none() { + to = Some(param); + } else { + return Err(usage_string(&executable_str)); + } + continue; + } + if param == "-b" || param == "--print-bytes" { + params.print_bytes = true; + continue; + } + if param == "-l" || param == "--verbose" { + params.verbose = true; + continue; + } + if param == "-lb" || param == "-bl" { + params.print_bytes = true; + params.verbose = true; + continue; + } + + let param_str = param.to_string_lossy().to_string(); + if param == "-n" || param_str.starts_with("--bytes=") { + let max_bytes = if param == "-n" { + opts.next() + .ok_or_else(|| usage_string(&executable_str))? + .to_string_lossy() + .to_string() + } else { + let (_, arg) = param_str.split_once('=').unwrap(); + arg.to_string() + }; + let max_bytes = match max_bytes.parse::() { + Ok(num) => num, + // TODO limit to MAX is dangerous, this should become an error like in GNU cmp. + Err(e) if *e.kind() == std::num::IntErrorKind::PosOverflow => BytesLimitU64::MAX, + Err(_) => { + return Err(format!( + "{executable_str}: invalid --bytes value '{max_bytes}'" + )) + } + }; + params.max_bytes = Some(max_bytes); + continue; + } + if param == "-i" || param_str.starts_with("--ignore-initial=") { + let skip_desc = if param == "-i" { + opts.next() + .ok_or_else(|| usage_string(&executable_str))? + .to_string_lossy() + .to_string() + } else { + let (_, arg) = param_str.split_once('=').unwrap(); + arg.to_string() + }; + let (skip_a, skip_b) = if let Some((skip_a, skip_b)) = skip_desc.split_once(':') { + ( + parse_skip(skip_a, &skip_desc)?, + parse_skip(skip_b, &skip_desc)?, + ) + } else { + let skip = parse_skip(&skip_desc, &skip_desc)?; + (skip, skip) + }; + params.skip_a = Some(skip_a); + params.skip_b = Some(skip_b); + continue; + } + if param == "-s" || param == "--quiet" || param == "--silent" { + params.quiet = true; + continue; + } + if param == "--help" { + println!("{}", usage_string(&executable_str)); + std::process::exit(0); + } + if param_str.starts_with('-') { + return Err(format!("unrecognized option '{}'", param.to_string_lossy())); + } + if from.is_none() { + from = Some(param); + } else if to.is_none() { + to = Some(param); + } else if skip_pos1.is_none() { + skip_pos1 = Some(parse_skip(¶m_str, ¶m_str)?); + } else if skip_pos2.is_none() { + skip_pos2 = Some(parse_skip(¶m_str, ¶m_str)?); + } else { + return Err(usage_string(&executable_str)); + } + } + + // Do as GNU cmp, and completely disable printing if we are + // outputting to /dev/null. + #[cfg(not(target_os = "windows"))] + if is_stdout_dev_null() { + params.quiet = true; + params.verbose = false; + params.print_bytes = false; + } + + if params.quiet && params.verbose { + return Err(format!( + "{executable_str}: options -l and -s are incompatible" + )); + } + + params.from = if let Some(from) = from { + from + } else if let Some(param) = opts.next() { + param + } else { + return Err(usage_string(&executable_str)); + }; + params.to = if let Some(to) = to { + to + } else if let Some(param) = opts.next() { + param + } else { + OsString::from("-") + }; + + // GNU cmp ignores positional skip arguments if -i is provided. + if params.skip_a.is_none() { + if skip_pos1.is_some() { + params.skip_a = skip_pos1; + } else if let Some(param) = opts.next() { + let param_str = param.to_string_lossy().to_string(); + params.skip_a = Some(parse_skip(¶m_str, ¶m_str)?); + } + }; + if params.skip_b.is_none() { + if skip_pos2.is_some() { + params.skip_b = skip_pos2; + } else if let Some(param) = opts.next() { + let param_str = param.to_string_lossy().to_string(); + params.skip_b = Some(parse_skip(¶m_str, ¶m_str)?); + } + } + + Ok(params) +} + +fn prepare_reader( + path: &OsString, + skip: &Option, + params: &Params, +) -> Result, String> { + let mut reader: Box = if path == "-" { + Box::new(BufReader::new(io::stdin())) + } else { + let file = fs::File::open(path) + .map_err(|e| format_failure_to_read_input_file(¶ms.executable, path, &e))?; + Box::new(BufReader::new(file)) + }; + + if let Some(skip) = skip { + // cast as u64 must remain, because value of IgnInit data type could be changed. + io::copy(&mut reader.by_ref().take(*skip), &mut io::sink()) + .map_err(|e| format_failure_to_read_input_file(¶ms.executable, path, &e))?; + } + + Ok(reader) +} + +#[derive(Debug)] +pub enum Cmp { + Equal, + Different, +} + +pub fn cmp(params: &Params) -> Result { + let mut from = prepare_reader(¶ms.from, ¶ms.skip_a, params)?; + let mut to = prepare_reader(¶ms.to, ¶ms.skip_b, params)?; + + let mut offset_width = params.max_bytes.unwrap_or(BytesLimitU64::MAX); + + if let (Ok(a_meta), Ok(b_meta)) = (fs::metadata(¶ms.from), fs::metadata(¶ms.to)) { + #[cfg(not(target_os = "windows"))] + let (a_size, b_size) = (a_meta.len(), b_meta.len()); + + #[cfg(target_os = "windows")] + let (a_size, b_size) = (a_meta.file_size(), b_meta.file_size()); + + // If the files have different sizes, we already know they are not identical. If we have not + // been asked to show even the first difference, we can quit early. + if params.quiet && a_size != b_size { + return Ok(Cmp::Different); + } + + let smaller = cmp::min(a_size, b_size) as BytesLimitU64; + offset_width = cmp::min(smaller, offset_width); + } + + let offset_width = 1 + offset_width.checked_ilog10().unwrap_or(1) as usize; + + // Capacity calc: at_byte width + 2 x 3-byte octal numbers + 2 x 4-byte value + 4 spaces + let mut output = Vec::::with_capacity(offset_width + 3 * 2 + 4 * 2 + 4); + + let mut at_byte: BytesLimitU64 = 1; + let mut at_line: u64 = 1; + let mut start_of_line = true; + let mut stdout = BufWriter::new(io::stdout().lock()); + let mut compare = Cmp::Equal; + loop { + // Fill up our buffers. + let from_buf = from + .fill_buf() + .map_err(|e| format_failure_to_read_input_file(¶ms.executable, ¶ms.from, &e))?; + + let to_buf = to + .fill_buf() + .map_err(|e| format_failure_to_read_input_file(¶ms.executable, ¶ms.to, &e))?; + + // Check for EOF conditions. + if from_buf.is_empty() && to_buf.is_empty() { + break; + } + + if from_buf.is_empty() || to_buf.is_empty() { + let eof_on = if from_buf.is_empty() { + ¶ms.from.to_string_lossy() + } else { + ¶ms.to.to_string_lossy() + }; + + report_eof(at_byte, at_line, start_of_line, eof_on, params); + return Ok(Cmp::Different); + } + + // Fast path - for long files in which almost all bytes are the same we + // can do a direct comparison to let the compiler optimize. + let consumed = std::cmp::min(from_buf.len(), to_buf.len()); + if from_buf[..consumed] == to_buf[..consumed] { + let last = from_buf[..consumed].last().unwrap(); + + at_byte += consumed as BytesLimitU64; + at_line += (from_buf[..consumed].iter().filter(|&c| *c == b'\n').count()) as u64; + + start_of_line = *last == b'\n'; + + if let Some(max_bytes) = params.max_bytes { + if at_byte > max_bytes { + break; + } + } + + from.consume(consumed); + to.consume(consumed); + + continue; + } + + // Iterate over the buffers, the zip iterator will stop us as soon as the + // first one runs out. + for (&from_byte, &to_byte) in from_buf.iter().zip(to_buf.iter()) { + if from_byte != to_byte { + compare = Cmp::Different; + + if params.verbose { + format_verbose_difference( + from_byte, + to_byte, + at_byte, + offset_width, + &mut output, + params, + )?; + stdout.write_all(output.as_slice()).map_err(|e| { + format!( + "{}: error printing output: {e}", + params.executable.to_string_lossy() + ) + })?; + output.clear(); + } else { + report_difference(from_byte, to_byte, at_byte, at_line, params); + return Ok(Cmp::Different); + } + } + + start_of_line = from_byte == b'\n'; + if start_of_line { + at_line += 1; + } + + at_byte += 1; + + if let Some(max_bytes) = params.max_bytes { + if at_byte > max_bytes { + break; + } + } + } + + // Notify our readers about the bytes we went over. + from.consume(consumed); + to.consume(consumed); + } + + Ok(compare) +} + +// Exit codes are documented at +// https://www.gnu.org/software/diffutils/manual/html_node/Invoking-cmp.html +// An exit status of 0 means no differences were found, +// 1 means some differences were found, +// and 2 means trouble. +pub fn main(opts: Peekable) -> ExitCode { + let params = match parse_params(opts) { + Ok(param) => param, + Err(e) => { + eprintln!("{e}"); + return ExitCode::from(2); + } + }; + + if params.from == "-" && params.to == "-" + || same_file::is_same_file(¶ms.from, ¶ms.to).unwrap_or(false) + { + return ExitCode::SUCCESS; + } + + match cmp(¶ms) { + Ok(Cmp::Equal) => ExitCode::SUCCESS, + Ok(Cmp::Different) => ExitCode::from(1), + Err(e) => { + if !params.quiet { + eprintln!("{e}"); + } + ExitCode::from(2) + } + } +} + +#[inline] +fn format_octal(byte: u8, buf: &mut [u8; 3]) -> &str { + *buf = *b" 0"; + + let mut num = byte; + let mut idx = 2; // Start at the last position in the buffer + + // Generate octal digits + while num > 0 { + buf[idx] = b'0' + num % 8; + num /= 8; + idx = idx.saturating_sub(1); + } + + // SAFETY: the operations we do above always land within ascii range. + unsafe { std::str::from_utf8_unchecked(&buf[..]) } +} + +#[inline] +fn write_visible_byte(output: &mut Vec, byte: u8) -> usize { + match byte { + // Control characters: ^@, ^A, ..., ^_ + 0..=31 => { + output.push(b'^'); + output.push(byte + 64); + 2 + } + // Printable ASCII (space through ~) + 32..=126 => { + output.push(byte); + 1 + } + // DEL: ^? + 127 => { + output.extend_from_slice(b"^?"); + 2 + } + // High bytes with control equivalents: M-^@, M-^A, ..., M-^_ + 128..=159 => { + output.push(b'M'); + output.push(b'-'); + output.push(b'^'); + output.push(byte - 64); + 4 + } + // High bytes: M-, M-!, ..., M-~ + 160..=254 => { + output.push(b'M'); + output.push(b'-'); + output.push(byte - 128); + 3 + } + // Byte 255: M-^? + 255 => { + output.extend_from_slice(b"M-^?"); + 4 + } + } +} + +/// Writes a byte in visible form with right-padding to 4 spaces. +#[inline] +fn write_visible_byte_padded(output: &mut Vec, byte: u8) { + const SPACES: &[u8] = b" "; + const WIDTH: usize = SPACES.len(); + + let display_width = write_visible_byte(output, byte); + + // Add right-padding spaces + let padding = WIDTH.saturating_sub(display_width); + output.extend_from_slice(&SPACES[..padding]); +} + +/// Formats a byte as a visible string (for non-performance-critical path) +#[inline] +fn format_visible_byte(byte: u8) -> String { + let mut result = Vec::with_capacity(4); + write_visible_byte(&mut result, byte); + // SAFETY: the checks and shifts in write_visible_byte match what cat and GNU + // cmp do to ensure characters fall inside the ascii range. + unsafe { String::from_utf8_unchecked(result) } +} + +// This function has been optimized to not use the Rust fmt system, which +// leads to a massive speed up when processing large files: cuts the time +// for comparing 2 ~36MB completely different files in half on an M1 Max. +#[inline] +fn format_verbose_difference( + from_byte: u8, + to_byte: u8, + at_byte: BytesLimitU64, + offset_width: usize, + output: &mut Vec, + params: &Params, +) -> Result<(), String> { + assert!(!params.quiet); + + let mut at_byte_buf = itoa::Buffer::new(); + let mut from_oct = [0u8; 3]; // for octal conversions + let mut to_oct = [0u8; 3]; + + if params.print_bytes { + // "{:>width$} {:>3o} {:4} {:>3o} {}", + let at_byte_str = at_byte_buf.format(at_byte); + let at_byte_padding = offset_width.saturating_sub(at_byte_str.len()); + + for _ in 0..at_byte_padding { + output.push(b' ') + } + + output.extend_from_slice(at_byte_str.as_bytes()); + + output.push(b' '); + + output.extend_from_slice(format_octal(from_byte, &mut from_oct).as_bytes()); + + output.push(b' '); + + write_visible_byte_padded(output, from_byte); + + output.push(b' '); + + output.extend_from_slice(format_octal(to_byte, &mut to_oct).as_bytes()); + + output.push(b' '); + + write_visible_byte(output, to_byte); + + output.push(b'\n'); + } else { + // "{:>width$} {:>3o} {:>3o}" + let at_byte_str = at_byte_buf.format(at_byte); + let at_byte_padding = offset_width.saturating_sub(at_byte_str.len()); + + for _ in 0..at_byte_padding { + output.push(b' ') + } + + output.extend_from_slice(at_byte_str.as_bytes()); + + output.push(b' '); + + output.extend_from_slice(format_octal(from_byte, &mut from_oct).as_bytes()); + + output.push(b' '); + + output.extend_from_slice(format_octal(to_byte, &mut to_oct).as_bytes()); + + output.push(b'\n'); + } + + Ok(()) +} + +#[inline] +fn report_eof( + at_byte: BytesLimitU64, + at_line: u64, + start_of_line: bool, + eof_on: &str, + params: &Params, +) { + if params.quiet { + return; + } + + if at_byte == 1 { + eprintln!( + "{}: EOF on '{}' which is empty", + params.executable.to_string_lossy(), + eof_on + ); + } else if params.verbose { + eprintln!( + "{}: EOF on '{}' after byte {}", + params.executable.to_string_lossy(), + eof_on, + at_byte - 1, + ); + } else if start_of_line { + eprintln!( + "{}: EOF on '{}' after byte {}, line {}", + params.executable.to_string_lossy(), + eof_on, + at_byte - 1, + at_line - 1 + ); + } else { + eprintln!( + "{}: EOF on '{}' after byte {}, in line {}", + params.executable.to_string_lossy(), + eof_on, + at_byte - 1, + at_line + ); + } +} + +fn is_posix_locale() -> bool { + let locale = if let Ok(locale) = env::var("LC_ALL") { + locale + } else if let Ok(locale) = env::var("LC_MESSAGES") { + locale + } else if let Ok(locale) = env::var("LANG") { + locale + } else { + "C".to_string() + }; + + locale == "C" || locale == "POSIX" +} + +#[inline] +fn report_difference( + from_byte: u8, + to_byte: u8, + at_byte: BytesLimitU64, + at_line: u64, + params: &Params, +) { + if params.quiet { + return; + } + + let term = if is_posix_locale() && !params.print_bytes { + "char" + } else { + "byte" + }; + print!( + "{} {} differ: {term} {}, line {}", + params.from.to_string_lossy(), + params.to.to_string_lossy(), + at_byte, + at_line + ); + if params.print_bytes { + let char_width = if to_byte >= 0x7F { 2 } else { 1 }; + print!( + " is {:>3o} {:char_width$} {:>3o} {:char_width$}", + from_byte, + format_visible_byte(from_byte), + to_byte, + format_visible_byte(to_byte) + ); + } + println!(); +} + +#[cfg(test)] +mod tests { + use super::*; + fn os(s: &str) -> OsString { + OsString::from(s) + } + + #[test] + fn positional() { + assert_eq!( + Ok(Params { + executable: os("cmp"), + from: os("foo"), + to: os("bar"), + ..Default::default() + }), + parse_params([os("cmp"), os("foo"), os("bar")].iter().cloned().peekable()) + ); + + assert_eq!( + Ok(Params { + executable: os("cmp"), + from: os("foo"), + to: os("-"), + ..Default::default() + }), + parse_params([os("cmp"), os("foo")].iter().cloned().peekable()) + ); + + assert_eq!( + Ok(Params { + executable: os("cmp"), + from: os("foo"), + to: os("--help"), + ..Default::default() + }), + parse_params( + [os("cmp"), os("foo"), os("--"), os("--help")] + .iter() + .cloned() + .peekable() + ) + ); + + assert_eq!( + Ok(Params { + executable: os("cmp"), + from: os("foo"), + to: os("bar"), + skip_a: Some(1), + skip_b: None, + ..Default::default() + }), + parse_params( + [os("cmp"), os("foo"), os("bar"), os("1")] + .iter() + .cloned() + .peekable() + ) + ); + + assert_eq!( + Ok(Params { + executable: os("cmp"), + from: os("foo"), + to: os("bar"), + skip_a: Some(1), + skip_b: Some(SkipU64::MAX), + ..Default::default() + }), + parse_params( + [os("cmp"), os("foo"), os("bar"), os("1"), os("2Y")] + .iter() + .cloned() + .peekable() + ) + ); + + // Bad positional arguments. + assert_eq!( + Err("Usage: cmp ".to_string()), + parse_params( + [os("cmp"), os("foo"), os("bar"), os("1"), os("2"), os("3")] + .iter() + .cloned() + .peekable() + ) + ); + assert_eq!( + Err("Usage: cmp ".to_string()), + parse_params([os("cmp")].iter().cloned().peekable()) + ); + } + + #[test] + fn execution_modes() { + let print_bytes = Params { + executable: os("cmp"), + from: os("foo"), + to: os("bar"), + print_bytes: true, + ..Default::default() + }; + assert_eq!( + Ok(print_bytes.clone()), + parse_params( + [os("cmp"), os("-b"), os("foo"), os("bar")] + .iter() + .cloned() + .peekable() + ) + ); + assert_eq!( + Ok(print_bytes), + parse_params( + [os("cmp"), os("--print-bytes"), os("foo"), os("bar")] + .iter() + .cloned() + .peekable() + ) + ); + + let verbose = Params { + executable: os("cmp"), + from: os("foo"), + to: os("bar"), + verbose: true, + ..Default::default() + }; + assert_eq!( + Ok(verbose.clone()), + parse_params( + [os("cmp"), os("-l"), os("foo"), os("bar")] + .iter() + .cloned() + .peekable() + ) + ); + assert_eq!( + Ok(verbose), + parse_params( + [os("cmp"), os("--verbose"), os("foo"), os("bar")] + .iter() + .cloned() + .peekable() + ) + ); + + let verbose_and_print_bytes = Params { + executable: os("cmp"), + from: os("foo"), + to: os("bar"), + print_bytes: true, + verbose: true, + ..Default::default() + }; + assert_eq!( + Ok(verbose_and_print_bytes.clone()), + parse_params( + [os("cmp"), os("-l"), os("-b"), os("foo"), os("bar")] + .iter() + .cloned() + .peekable() + ) + ); + assert_eq!( + Ok(verbose_and_print_bytes.clone()), + parse_params( + [os("cmp"), os("-lb"), os("foo"), os("bar")] + .iter() + .cloned() + .peekable() + ) + ); + assert_eq!( + Ok(verbose_and_print_bytes), + parse_params( + [os("cmp"), os("-bl"), os("foo"), os("bar")] + .iter() + .cloned() + .peekable() + ) + ); + + assert_eq!( + Ok(Params { + executable: os("cmp"), + from: os("foo"), + to: os("bar"), + quiet: true, + ..Default::default() + }), + parse_params( + [os("cmp"), os("-s"), os("foo"), os("bar")] + .iter() + .cloned() + .peekable() + ) + ); + + // Some options do not mix. + assert_eq!( + Err("cmp: options -l and -s are incompatible".to_string()), + parse_params( + [os("cmp"), os("-l"), os("-s"), os("foo"), os("bar")] + .iter() + .cloned() + .peekable() + ) + ); + } + + #[test] + fn max_bytes() { + let max_bytes = Params { + executable: os("cmp"), + from: os("foo"), + to: os("bar"), + max_bytes: Some(1), + ..Default::default() + }; + assert_eq!( + Ok(max_bytes.clone()), + parse_params( + [os("cmp"), os("-n"), os("1"), os("foo"), os("bar")] + .iter() + .cloned() + .peekable() + ) + ); + assert_eq!( + Ok(max_bytes), + parse_params( + [os("cmp"), os("--bytes=1"), os("foo"), os("bar")] + .iter() + .cloned() + .peekable() + ) + ); + + assert_eq!( + Ok(Params { + executable: os("cmp"), + from: os("foo"), + to: os("bar"), + max_bytes: Some(BytesLimitU64::MAX), + ..Default::default() + }), + parse_params( + [ + os("cmp"), + os("--bytes=99999999999999999999999999999999999999999999999999999999999"), + os("foo"), + os("bar") + ] + .iter() + .cloned() + .peekable() + ) + ); + + // Failure case + // TODO This is actually fine in GNU cmp. --bytes does not have a unit parser yet. + assert_eq!( + Err("cmp: invalid --bytes value '1K'".to_string()), + parse_params( + [os("cmp"), os("--bytes=1K"), os("foo"), os("bar")] + .iter() + .cloned() + .peekable() + ) + ); + } + + #[test] + fn skips() { + let skips = Params { + executable: os("cmp"), + from: os("foo"), + to: os("bar"), + skip_a: Some(1), + skip_b: Some(1), + ..Default::default() + }; + assert_eq!( + Ok(skips.clone()), + parse_params( + [os("cmp"), os("-i"), os("1"), os("foo"), os("bar")] + .iter() + .cloned() + .peekable() + ) + ); + assert_eq!( + Ok(skips), + parse_params( + [os("cmp"), os("--ignore-initial=1"), os("foo"), os("bar")] + .iter() + .cloned() + .peekable() + ) + ); + + assert_eq!( + Ok(Params { + executable: os("cmp"), + from: os("foo"), + to: os("bar"), + skip_a: Some(SkipU64::MAX), + skip_b: Some(SkipU64::MAX), + ..Default::default() + }), + parse_params( + [ + os("cmp"), + os("-i"), + os("99999999999999999999999999999999999999999999999999999999999"), + os("foo"), + os("bar") + ] + .iter() + .cloned() + .peekable() + ) + ); + + assert_eq!( + Ok(Params { + executable: os("cmp"), + from: os("foo"), + to: os("bar"), + skip_a: Some(1), + skip_b: Some(2), + ..Default::default() + }), + parse_params( + [os("cmp"), os("--ignore-initial=1:2"), os("foo"), os("bar")] + .iter() + .cloned() + .peekable() + ) + ); + + assert_eq!( + Ok(Params { + executable: os("cmp"), + from: os("foo"), + to: os("bar"), + skip_a: Some(1_000_000_000), + #[cfg(target_pointer_width = "32")] + skip_b: Some((2_147_483_647.5 * 2.0) as usize), + #[cfg(target_pointer_width = "64")] + skip_b: Some(1_152_921_504_606_846_976 * 2), + ..Default::default() + }), + parse_params( + [ + os("cmp"), + os("--ignore-initial=1GB:2E"), + os("foo"), + os("bar") + ] + .iter() + .cloned() + .peekable() + ) + ); + + // All special suffixes. + for (i, suffixes) in [ + ["kB", "K"], + ["MB", "M"], + ["GB", "G"], + ["TB", "T"], + ["PB", "P"], + ["EB", "E"], + ["ZB", "Z"], + ["YB", "Y"], + ] + .iter() + .enumerate() + { + let values = [ + (1_000 as SkipU64) + .checked_pow((i + 1) as u32) + .unwrap_or(SkipU64::MAX), + (1024 as SkipU64) + .checked_pow((i + 1) as u32) + .unwrap_or(SkipU64::MAX), + ]; + for (j, v) in values.iter().enumerate() { + assert_eq!( + Ok(Params { + executable: os("cmp"), + from: os("foo"), + to: os("bar"), + skip_a: Some(*v), + skip_b: Some(2), + ..Default::default() + }), + parse_params( + [ + os("cmp"), + os("-i"), + os(&format!("1{}:2", suffixes[j])), + os("foo"), + os("bar"), + ] + .iter() + .cloned() + .peekable() + ) + ); + } + } + + // Ignores positional arguments when -i is provided. + assert_eq!( + Ok(Params { + executable: os("cmp"), + from: os("foo"), + to: os("bar"), + skip_a: Some(1), + skip_b: Some(2), + ..Default::default() + }), + parse_params( + [ + os("cmp"), + os("-i"), + os("1:2"), + os("foo"), + os("bar"), + os("3"), + os("4") + ] + .iter() + .cloned() + .peekable() + ) + ); + + // Failure cases + assert_eq!( + Err("cmp: invalid --ignore-initial value '1mb'".to_string()), + parse_params( + [os("cmp"), os("--ignore-initial=1mb"), os("foo"), os("bar")] + .iter() + .cloned() + .peekable() + ) + ); + assert_eq!( + Err("cmp: invalid --ignore-initial value '1:2:3'".to_string()), + parse_params( + [ + os("cmp"), + os("--ignore-initial=1:2:3"), + os("foo"), + os("bar") + ] + .iter() + .cloned() + .peekable() + ) + ); + assert_eq!( + Err("cmp: invalid --ignore-initial value '-1'".to_string()), + parse_params( + [os("cmp"), os("--ignore-initial=-1"), os("foo"), os("bar")] + .iter() + .cloned() + .peekable() + ) + ); + } +} diff --git a/userland/diffutils/src/context_diff.rs b/userland/diffutils/src/context_diff.rs new file mode 100644 index 0000000..a37e776 --- /dev/null +++ b/userland/diffutils/src/context_diff.rs @@ -0,0 +1,998 @@ +// This file is part of the uutils diffutils package. +// +// For the full copyright and license information, please view the LICENSE-* +// files that was distributed with this source code. + +use std::collections::VecDeque; +use std::io::Write; + +use crate::params::Params; +use crate::utils::do_write_line; +use crate::utils::get_modification_time; + +#[derive(Debug, PartialEq)] +pub enum DiffLine { + Context(Vec), + Change(Vec), + Add(Vec), +} + +#[derive(Debug, PartialEq)] +struct Mismatch { + pub line_number_expected: usize, + pub line_number_actual: usize, + pub expected: Vec, + pub actual: Vec, + pub expected_missing_nl: bool, + pub actual_missing_nl: bool, + pub expected_all_context: bool, + pub actual_all_context: bool, +} + +impl Mismatch { + fn new(line_number_expected: usize, line_number_actual: usize) -> Mismatch { + Mismatch { + line_number_expected, + line_number_actual, + expected: Vec::new(), + actual: Vec::new(), + expected_missing_nl: false, + actual_missing_nl: false, + expected_all_context: false, + actual_all_context: false, + } + } +} + +/// A run of changed lines with no common line between them. +struct ChangeBlock { + expected_start: usize, + actual_start: usize, + deletes: bool, + inserts: bool, +} + +impl ChangeBlock { + fn new(mismatch: &Mismatch) -> Self { + Self { + expected_start: mismatch.expected.len(), + actual_start: mismatch.actual.len(), + deletes: false, + inserts: false, + } + } + + /// Opens a block at the end of `mismatch` if none is open, and records + /// whether it deletes or inserts lines. + fn mark(block: &mut Option, mismatch: &Mismatch, deletes: bool, inserts: bool) { + let block = block.get_or_insert_with(|| Self::new(mismatch)); + block.deletes |= deletes; + block.inserts |= inserts; + } + + fn close(block: &mut Option, mismatch: &mut Mismatch) { + if let Some(block) = block.take() { + block.finish(mismatch); + } + } + + /// Like GNU diff, a block that both deletes and inserts lines marks all of + /// them as changed (`!`), not only as many as pair up. + fn finish(self, mismatch: &mut Mismatch) { + if !(self.deletes && self.inserts) { + return; + } + let expected = mismatch.expected[self.expected_start..].iter_mut(); + let actual = mismatch.actual[self.actual_start..].iter_mut(); + for line in expected.chain(actual) { + if let DiffLine::Add(content) = line { + *line = DiffLine::Change(std::mem::take(content)); + } + } + } +} + +// Produces a diff between the expected output and actual output. +fn make_diff( + expected: &[u8], + actual: &[u8], + context_size: usize, + stop_early: bool, +) -> Vec { + let mut line_number_expected = 1; + let mut line_number_actual = 1; + let mut context_queue: VecDeque<&[u8]> = VecDeque::with_capacity(context_size); + // Like GNU diff, keep changes in one hunk while at most twice the context + // size of unchanged lines separates them. + let max_gap = 2 * context_size; + let mut lines_since_mismatch = max_gap + 1; + let mut results = Vec::new(); + let mut mismatch = Mismatch::new(0, 0); + + let mut expected_lines: Vec<&[u8]> = expected.split(|&c| c == b'\n').collect(); + let mut actual_lines: Vec<&[u8]> = actual.split(|&c| c == b'\n').collect(); + + debug_assert_eq!(b"".split(|&c| c == b'\n').count(), 1); + // ^ means that underflow here is impossible + let expected_lines_count = expected_lines.len() - 1; + let actual_lines_count = actual_lines.len() - 1; + + if expected_lines.last() == Some(&&b""[..]) { + expected_lines.pop(); + } + + if actual_lines.last() == Some(&&b""[..]) { + actual_lines.pop(); + } + + let mut block: Option = None; + + for result in diff::slice(&expected_lines, &actual_lines) { + match result { + diff::Result::Left(str) => { + if lines_since_mismatch > max_gap { + results.push(mismatch); + mismatch = Mismatch::new( + line_number_expected - context_queue.len(), + line_number_actual - context_queue.len(), + ); + } + + while let Some(line) = context_queue.pop_front() { + mismatch.expected.push(DiffLine::Context(line.to_vec())); + mismatch.actual.push(DiffLine::Context(line.to_vec())); + } + + ChangeBlock::mark(&mut block, &mismatch, true, false); + mismatch.expected.push(DiffLine::Add(str.to_vec())); + if line_number_expected > expected_lines_count { + mismatch.expected_missing_nl = true; + } + line_number_expected += 1; + lines_since_mismatch = 0; + } + diff::Result::Right(str) => { + if lines_since_mismatch > max_gap { + results.push(mismatch); + mismatch = Mismatch::new( + line_number_expected - context_queue.len(), + line_number_actual - context_queue.len(), + ); + } + + while let Some(line) = context_queue.pop_front() { + mismatch.expected.push(DiffLine::Context(line.to_vec())); + mismatch.actual.push(DiffLine::Context(line.to_vec())); + } + + ChangeBlock::mark(&mut block, &mismatch, false, true); + mismatch.actual.push(DiffLine::Add(str.to_vec())); + if line_number_actual > actual_lines_count { + mismatch.actual_missing_nl = true; + } + line_number_actual += 1; + lines_since_mismatch = 0; + } + diff::Result::Both(str, _) => { + // if one of them is missing a newline and the other isn't, then they don't actually match + if (line_number_actual > actual_lines_count) + && (line_number_expected > expected_lines_count) + { + ChangeBlock::close(&mut block, &mut mismatch); + if context_queue.len() < context_size { + while let Some(line) = context_queue.pop_front() { + mismatch.expected.push(DiffLine::Context(line.to_vec())); + mismatch.actual.push(DiffLine::Context(line.to_vec())); + } + if lines_since_mismatch < context_size { + mismatch.expected.push(DiffLine::Context(str.to_vec())); + mismatch.actual.push(DiffLine::Context(str.to_vec())); + mismatch.expected_missing_nl = true; + mismatch.actual_missing_nl = true; + } + } + lines_since_mismatch = 0; + } else if line_number_actual > actual_lines_count { + if lines_since_mismatch > max_gap { + results.push(mismatch); + mismatch = Mismatch::new( + line_number_expected - context_queue.len(), + line_number_actual - context_queue.len(), + ); + } + while let Some(line) = context_queue.pop_front() { + mismatch.expected.push(DiffLine::Context(line.to_vec())); + mismatch.actual.push(DiffLine::Context(line.to_vec())); + } + ChangeBlock::mark(&mut block, &mismatch, true, true); + mismatch.expected.push(DiffLine::Change(str.to_vec())); + mismatch.actual.push(DiffLine::Change(str.to_vec())); + mismatch.actual_missing_nl = true; + lines_since_mismatch = 0; + } else if line_number_expected > expected_lines_count { + if lines_since_mismatch > max_gap { + results.push(mismatch); + mismatch = Mismatch::new( + line_number_expected - context_queue.len(), + line_number_actual - context_queue.len(), + ); + } + while let Some(line) = context_queue.pop_front() { + mismatch.expected.push(DiffLine::Context(line.to_vec())); + mismatch.actual.push(DiffLine::Context(line.to_vec())); + } + ChangeBlock::mark(&mut block, &mismatch, true, true); + mismatch.expected.push(DiffLine::Change(str.to_vec())); + mismatch.expected_missing_nl = true; + mismatch.actual.push(DiffLine::Change(str.to_vec())); + lines_since_mismatch = 0; + } else { + ChangeBlock::close(&mut block, &mut mismatch); + debug_assert!(context_queue.len() <= context_size); + if context_queue.len() >= context_size { + let _ = context_queue.pop_front(); + } + if lines_since_mismatch < context_size { + mismatch.expected.push(DiffLine::Context(str.to_vec())); + mismatch.actual.push(DiffLine::Context(str.to_vec())); + } else if context_size > 0 { + context_queue.push_back(str); + } + lines_since_mismatch += 1; + } + line_number_expected += 1; + line_number_actual += 1; + } + } + if stop_early && !results.is_empty() { + // Optimization: stop analyzing the files as soon as there are any differences + return results; + } + } + + if let Some(block) = block { + block.finish(&mut mismatch); + } + results.push(mismatch); + results.remove(0); + + if results.is_empty() && expected_lines_count != actual_lines_count { + let mut mismatch = Mismatch::new(expected_lines.len(), actual_lines.len()); + // empty diff and only expected lines has a missing line at end + if expected_lines_count != expected_lines.len() { + mismatch.expected.push(DiffLine::Change( + expected_lines + .pop() + .expect("can't be empty; produced by split()") + .to_vec(), + )); + mismatch.expected_missing_nl = true; + mismatch.actual.push(DiffLine::Change( + actual_lines + .pop() + .expect("can't be empty; produced by split()") + .to_vec(), + )); + results.push(mismatch); + } else if actual_lines_count != actual_lines.len() { + mismatch.expected.push(DiffLine::Change( + expected_lines + .pop() + .expect("can't be empty; produced by split()") + .to_vec(), + )); + mismatch.actual.push(DiffLine::Change( + actual_lines + .pop() + .expect("can't be empty; produced by split()") + .to_vec(), + )); + mismatch.actual_missing_nl = true; + results.push(mismatch); + } + } + + // hunks with pure context lines get truncated to empty + for mismatch in &mut results { + if !mismatch + .expected + .iter() + .any(|x| !matches!(&x, DiffLine::Context(_))) + { + mismatch.expected_all_context = true; + } + if !mismatch + .actual + .iter() + .any(|x| !matches!(&x, DiffLine::Context(_))) + { + mismatch.actual_all_context = true; + } + } + + results +} + +#[must_use] +pub fn diff(expected: &[u8], actual: &[u8], params: &Params) -> Vec { + let from_modified_time = get_modification_time(¶ms.from.to_string_lossy()); + let to_modified_time = get_modification_time(¶ms.to.to_string_lossy()); + let mut output = format!( + "*** {0}\t{1}\n--- {2}\t{3}\n", + params.from.to_string_lossy(), + from_modified_time, + params.to.to_string_lossy(), + to_modified_time + ) + .into_bytes(); + let diff_results = make_diff(expected, actual, params.context_count, params.brief); + if diff_results.is_empty() { + return Vec::new(); + } + if params.brief { + return output; + } + for result in diff_results { + let mut line_number_expected = result.line_number_expected; + let mut line_number_actual = result.line_number_actual; + let mut expected_count = result.expected.len(); + let mut actual_count = result.actual.len(); + if expected_count == 0 { + line_number_expected -= 1; + expected_count = 1; + } + if actual_count == 0 { + line_number_actual -= 1; + actual_count = 1; + } + let end_line_number_expected = expected_count + line_number_expected - 1; + let end_line_number_actual = actual_count + line_number_actual - 1; + let exp_start = if end_line_number_expected == line_number_expected { + String::new() + } else { + format!("{line_number_expected},") + }; + let act_start = if end_line_number_actual == line_number_actual { + String::new() + } else { + format!("{line_number_actual},") + }; + writeln!( + output, + "***************\n*** {exp_start}{end_line_number_expected} ****" + ) + .expect("write to Vec is infallible"); + if !result.expected_all_context { + for line in result.expected { + match line { + DiffLine::Context(e) => { + write!(output, " ").expect("write to Vec is infallible"); + do_write_line(&mut output, &e, params.expand_tabs, params.tabsize) + .expect("write to Vec is infallible"); + writeln!(output).unwrap(); + } + DiffLine::Change(e) => { + write!(output, "! ").expect("write to Vec is infallible"); + do_write_line(&mut output, &e, params.expand_tabs, params.tabsize) + .expect("write to Vec is infallible"); + writeln!(output).unwrap(); + } + DiffLine::Add(e) => { + write!(output, "- ").expect("write to Vec is infallible"); + do_write_line(&mut output, &e, params.expand_tabs, params.tabsize) + .expect("write to Vec is infallible"); + writeln!(output).unwrap(); + } + } + } + if result.expected_missing_nl { + writeln!(output, r"\ No newline at end of file") + .expect("write to Vec is infallible"); + } + } + writeln!(output, "--- {act_start}{end_line_number_actual} ----") + .expect("write to Vec is infallible"); + if !result.actual_all_context { + for line in result.actual { + match line { + DiffLine::Context(e) => { + write!(output, " ").expect("write to Vec is infallible"); + do_write_line(&mut output, &e, params.expand_tabs, params.tabsize) + .expect("write to Vec is infallible"); + writeln!(output).unwrap(); + } + DiffLine::Change(e) => { + write!(output, "! ").expect("write to Vec is infallible"); + do_write_line(&mut output, &e, params.expand_tabs, params.tabsize) + .expect("write to Vec is infallible"); + writeln!(output).unwrap(); + } + DiffLine::Add(e) => { + write!(output, "+ ").expect("write to Vec is infallible"); + do_write_line(&mut output, &e, params.expand_tabs, params.tabsize) + .expect("write to Vec is infallible"); + writeln!(output).unwrap(); + } + } + } + if result.actual_missing_nl { + writeln!(output, r"\ No newline at end of file") + .expect("write to Vec is infallible"); + } + } + } + output +} + +#[cfg(test)] +mod tests { + use super::*; + use pretty_assertions::assert_eq; + + use crate::utils::testcmds::PATCH_CMD; + + #[test] + fn test_permutations() { + // test all possible six-line files. + let target = "target/context-diff/"; + let _ = std::fs::create_dir(target); + for &a in &[0, 1, 2] { + for &b in &[0, 1, 2] { + for &c in &[0, 1, 2] { + for &d in &[0, 1, 2] { + for &e in &[0, 1, 2] { + for &f in &[0, 1, 2] { + use std::fs::{self, File}; + use std::io::Write; + let mut alef = Vec::new(); + let mut bet = Vec::new(); + alef.write_all(if a == 0 { b"a\n" } else { b"b\n" }) + .unwrap(); + if a != 2 { + bet.write_all(b"b\n").unwrap(); + } + alef.write_all(if b == 0 { b"c\n" } else { b"d\n" }) + .unwrap(); + if b != 2 { + bet.write_all(b"d\n").unwrap(); + } + alef.write_all(if c == 0 { b"e\n" } else { b"f\n" }) + .unwrap(); + if c != 2 { + bet.write_all(b"f\n").unwrap(); + } + alef.write_all(if d == 0 { b"g\n" } else { b"h\n" }) + .unwrap(); + if d != 2 { + bet.write_all(b"h\n").unwrap(); + } + alef.write_all(if e == 0 { b"i\n" } else { b"j\n" }) + .unwrap(); + if e != 2 { + bet.write_all(b"j\n").unwrap(); + } + alef.write_all(if f == 0 { b"k\n" } else { b"l\n" }) + .unwrap(); + if f != 2 { + bet.write_all(b"l\n").unwrap(); + } + // This test diff is intentionally reversed. + // We want it to turn the alef into bet. + let patched = &format!("{target}/alef"); + let diff = diff( + &alef, + &bet, + &Params { + from: patched.into(), + to: patched.into(), + context_count: 2, + ..Default::default() + }, + ); + File::create(format!("{target}/ab.diff")) + .unwrap() + .write_all(&diff) + .unwrap(); + let mut fa = File::create(format!("{target}/alef")).unwrap(); + fa.write_all(&alef[..]).unwrap(); + let mut fb = File::create(format!("{target}/bet")).unwrap(); + fb.write_all(&bet[..]).unwrap(); + let _ = fa; + let _ = fb; + let output = PATCH_CMD + .new() + .arg("-p0") + .arg("--context") + .stdin(File::open(format!("{target}/ab.diff")).unwrap()) + .output() + .unwrap(); + assert!(output.status.success(), "{output:?}"); + //println!("{}", String::from_utf8_lossy(&output.stdout)); + //println!("{}", String::from_utf8_lossy(&output.stderr)); + let alef = fs::read(format!("{target}/alef")).unwrap(); + assert_eq!(alef, bet); + } + } + } + } + } + } + } + + #[test] + fn test_permutations_empty_lines() { + let target = "target/context-diff/"; + // test all possible six-line files with missing newlines. + let _ = std::fs::create_dir(target); + for &a in &[0, 1, 2] { + for &b in &[0, 1, 2] { + for &c in &[0, 1, 2] { + for &d in &[0, 1, 2] { + for &e in &[0, 1, 2] { + for &f in &[0, 1, 2] { + use std::fs::{self, File}; + use std::io::Write; + let mut alef = Vec::new(); + let mut bet = Vec::new(); + alef.write_all(if a == 0 { b"\n" } else { b"b\n" }).unwrap(); + if a != 2 { + bet.write_all(b"b\n").unwrap(); + } + alef.write_all(if b == 0 { b"\n" } else { b"d\n" }).unwrap(); + if b != 2 { + bet.write_all(b"d\n").unwrap(); + } + alef.write_all(if c == 0 { b"\n" } else { b"f\n" }).unwrap(); + if c != 2 { + bet.write_all(b"f\n").unwrap(); + } + alef.write_all(if d == 0 { b"\n" } else { b"h\n" }).unwrap(); + if d != 2 { + bet.write_all(b"h\n").unwrap(); + } + alef.write_all(if e == 0 { b"\n" } else { b"j\n" }).unwrap(); + if e != 2 { + bet.write_all(b"j\n").unwrap(); + } + alef.write_all(if f == 0 { b"\n" } else { b"l\n" }).unwrap(); + if f != 2 { + bet.write_all(b"l\n").unwrap(); + } + // This test diff is intentionally reversed. + // We want it to turn the alef into bet. + let patched = &format!("{target}/alef_"); + let diff = diff( + &alef, + &bet, + &Params { + from: patched.into(), + to: patched.into(), + context_count: 2, + ..Default::default() + }, + ); + File::create(format!("{target}/ab_.diff")) + .unwrap() + .write_all(&diff) + .unwrap(); + let mut fa = File::create(format!("{target}/alef_")).unwrap(); + fa.write_all(&alef[..]).unwrap(); + let mut fb = File::create(format!("{target}/bet_")).unwrap(); + fb.write_all(&bet[..]).unwrap(); + let _ = fa; + let _ = fb; + let output = PATCH_CMD + .new() + .arg("-p0") + .arg("--context") + .stdin(File::open(format!("{target}/ab_.diff")).unwrap()) + .output() + .unwrap(); + assert!(output.status.success(), "{output:?}"); + //println!("{}", String::from_utf8_lossy(&output.stdout)); + //println!("{}", String::from_utf8_lossy(&output.stderr)); + let alef = fs::read(format!("{target}/alef_")).unwrap(); + assert_eq!(alef, bet); + } + } + } + } + } + } + } + + #[test] + fn test_permutations_missing_lines() { + let target = "target/context-diff/"; + // test all possible six-line files. + let _ = std::fs::create_dir(target); + for &a in &[0, 1, 2] { + for &b in &[0, 1, 2] { + for &c in &[0, 1, 2] { + for &d in &[0, 1, 2] { + for &e in &[0, 1, 2] { + for &f in &[0, 1, 2] { + use std::fs::{self, File}; + use std::io::Write; + let mut alef = Vec::new(); + let mut bet = Vec::new(); + alef.write_all(if a == 0 { b"a\n" } else { b"" }).unwrap(); + if a != 2 { + bet.write_all(b"b\n").unwrap(); + } + alef.write_all(if b == 0 { b"c\n" } else { b"" }).unwrap(); + if b != 2 { + bet.write_all(b"d\n").unwrap(); + } + alef.write_all(if c == 0 { b"e\n" } else { b"" }).unwrap(); + if c != 2 { + bet.write_all(b"f\n").unwrap(); + } + alef.write_all(if d == 0 { b"g\n" } else { b"" }).unwrap(); + if d != 2 { + bet.write_all(b"h\n").unwrap(); + } + alef.write_all(if e == 0 { b"i\n" } else { b"" }).unwrap(); + if e != 2 { + bet.write_all(b"j\n").unwrap(); + } + alef.write_all(if f == 0 { b"k\n" } else { b"" }).unwrap(); + if f != 2 { + bet.write_all(b"l\n").unwrap(); + } + if alef.is_empty() && bet.is_empty() { + continue; + }; + // This test diff is intentionally reversed. + // We want it to turn the alef into bet. + let patched = &format!("{target}/alefx"); + let diff = diff( + &alef, + &bet, + &Params { + from: patched.into(), + to: patched.into(), + context_count: 2, + ..Default::default() + }, + ); + File::create(format!("{target}/abx.diff")) + .unwrap() + .write_all(&diff) + .unwrap(); + let mut fa = File::create(format!("{target}/alefx")).unwrap(); + fa.write_all(&alef[..]).unwrap(); + let mut fb = File::create(format!("{target}/betx")).unwrap(); + fb.write_all(&bet[..]).unwrap(); + let _ = fa; + let _ = fb; + let output = PATCH_CMD + .new() + .arg("-p0") + .arg("--context") + .stdin(File::open(format!("{target}/abx.diff")).unwrap()) + .output() + .unwrap(); + assert!(output.status.success(), "{output:?}"); + //println!("{}", String::from_utf8_lossy(&output.stdout)); + //println!("{}", String::from_utf8_lossy(&output.stderr)); + let alef = fs::read(format!("{target}/alefx")).unwrap(); + assert_eq!(alef, bet); + } + } + } + } + } + } + } + + #[test] + fn test_permutations_reverse() { + let target = "target/context-diff/"; + // test all possible six-line files. + let _ = std::fs::create_dir(target); + for &a in &[0, 1, 2] { + for &b in &[0, 1, 2] { + for &c in &[0, 1, 2] { + for &d in &[0, 1, 2] { + for &e in &[0, 1, 2] { + for &f in &[0, 1, 2] { + use std::fs::{self, File}; + use std::io::Write; + let mut alef = Vec::new(); + let mut bet = Vec::new(); + alef.write_all(if a == 0 { b"a\n" } else { b"f\n" }) + .unwrap(); + if a != 2 { + bet.write_all(b"a\n").unwrap(); + } + alef.write_all(if b == 0 { b"b\n" } else { b"e\n" }) + .unwrap(); + if b != 2 { + bet.write_all(b"b\n").unwrap(); + } + alef.write_all(if c == 0 { b"c\n" } else { b"d\n" }) + .unwrap(); + if c != 2 { + bet.write_all(b"c\n").unwrap(); + } + alef.write_all(if d == 0 { b"d\n" } else { b"c\n" }) + .unwrap(); + if d != 2 { + bet.write_all(b"d\n").unwrap(); + } + alef.write_all(if e == 0 { b"e\n" } else { b"b\n" }) + .unwrap(); + if e != 2 { + bet.write_all(b"e\n").unwrap(); + } + alef.write_all(if f == 0 { b"f\n" } else { b"a\n" }) + .unwrap(); + if f != 2 { + bet.write_all(b"f\n").unwrap(); + } + // This test diff is intentionally reversed. + // We want it to turn the alef into bet. + let alefr_path = &format!("{target}/alefr"); + let diff = diff( + &alef, + &bet, + &Params { + from: alefr_path.into(), + to: alefr_path.into(), + context_count: 2, + ..Default::default() + }, + ); + File::create(format!("{target}/abr.diff")) + .unwrap() + .write_all(&diff) + .unwrap(); + let mut fa = File::create(format!("{target}/alefr")).unwrap(); + fa.write_all(&alef[..]).unwrap(); + let mut fb = File::create(format!("{target}/betr")).unwrap(); + fb.write_all(&bet[..]).unwrap(); + let _ = fa; + let _ = fb; + let output = PATCH_CMD + .new() + .arg("-p0") + .arg("--context") + .stdin(File::open(format!("{target}/abr.diff")).unwrap()) + .output() + .unwrap(); + assert!(output.status.success(), "{output:?}"); + //println!("{}", String::from_utf8_lossy(&output.stdout)); + //println!("{}", String::from_utf8_lossy(&output.stderr)); + let alef = fs::read(format!("{target}/alefr")).unwrap(); + assert_eq!(alef, bet); + } + } + } + } + } + } + } + + #[test] + fn test_stop_early() { + use crate::assert_diff_eq; + + let from_filename = "foo"; + let from = ["a", "b", "c", ""].join("\n"); + let to_filename = "bar"; + let to = ["a", "d", "c", ""].join("\n"); + + let diff_full = diff( + from.as_bytes(), + to.as_bytes(), + &Params { + from: from_filename.into(), + to: to_filename.into(), + ..Default::default() + }, + ); + + let expected_full = [ + "*** foo\tTIMESTAMP", + "--- bar\tTIMESTAMP", + "***************", + "*** 1,3 ****", + " a", + "! b", + " c", + "--- 1,3 ----", + " a", + "! d", + " c", + "", + ] + .join("\n"); + assert_diff_eq!(diff_full, expected_full); + + let diff_brief = diff( + from.as_bytes(), + to.as_bytes(), + &Params { + from: from_filename.into(), + to: to_filename.into(), + brief: true, + ..Default::default() + }, + ); + + let expected_brief = ["*** foo\tTIMESTAMP", "--- bar\tTIMESTAMP", ""].join("\n"); + assert_diff_eq!(diff_brief, expected_brief); + + let nodiff_full = diff( + from.as_bytes(), + from.as_bytes(), + &Params { + from: from_filename.into(), + to: to_filename.into(), + ..Default::default() + }, + ); + assert!(nodiff_full.is_empty()); + + let nodiff_brief = diff( + from.as_bytes(), + from.as_bytes(), + &Params { + from: from_filename.into(), + to: to_filename.into(), + brief: true, + ..Default::default() + }, + ); + assert!(nodiff_brief.is_empty()); + } + + #[test] + fn test_change_block_lines_all_changed() { + use crate::assert_diff_eq; + + let params = Params { + from: "foo".into(), + to: "bar".into(), + ..Default::default() + }; + + // Two lines replaced by one: both old lines are changed, not one deleted. + let from = ["a", "b", "c", "d", ""].join("\n"); + let to = ["a", "x", "d", ""].join("\n"); + let expected = [ + "*** foo\tTIMESTAMP", + "--- bar\tTIMESTAMP", + "***************", + "*** 1,4 ****", + " a", + "! b", + "! c", + " d", + "--- 1,3 ----", + " a", + "! x", + " d", + "", + ] + .join("\n"); + let output = diff(from.as_bytes(), to.as_bytes(), ¶ms); + assert_diff_eq!(output, expected); + + // One line replaced by three: all three new lines are changed. + let from = ["a", "b", ""].join("\n"); + let to = ["a", "x", "y", "z", ""].join("\n"); + let expected = [ + "*** foo\tTIMESTAMP", + "--- bar\tTIMESTAMP", + "***************", + "*** 1,2 ****", + " a", + "! b", + "--- 1,4 ----", + " a", + "! x", + "! y", + "! z", + "", + ] + .join("\n"); + let output = diff(from.as_bytes(), to.as_bytes(), ¶ms); + assert_diff_eq!(output, expected); + } + + #[test] + fn test_hunk_merging() { + use crate::assert_diff_eq; + + let params = Params { + from: "foo".into(), + to: "bar".into(), + context_count: 1, + ..Default::default() + }; + + // Changes separated by up to twice the context size share one hunk. + let from = ["a", "b", "c", "d", "e", "f", ""].join("\n"); + let to = ["a", "B", "c", "d", "E", "f", ""].join("\n"); + let expected = [ + "*** foo\tTIMESTAMP", + "--- bar\tTIMESTAMP", + "***************", + "*** 1,6 ****", + " a", + "! b", + " c", + " d", + "! e", + " f", + "--- 1,6 ----", + " a", + "! B", + " c", + " d", + "! E", + " f", + "", + ] + .join("\n"); + let output = diff(from.as_bytes(), to.as_bytes(), ¶ms); + assert_diff_eq!(output, expected); + + // One more unchanged line between them splits the hunk. + let from = ["a", "b", "c", "d", "e", "f", "g", ""].join("\n"); + let to = ["a", "B", "c", "d", "e", "F", "g", ""].join("\n"); + let expected = [ + "*** foo\tTIMESTAMP", + "--- bar\tTIMESTAMP", + "***************", + "*** 1,3 ****", + " a", + "! b", + " c", + "--- 1,3 ----", + " a", + "! B", + " c", + "***************", + "*** 5,7 ****", + " e", + "! f", + " g", + "--- 5,7 ----", + " e", + "! F", + " g", + "", + ] + .join("\n"); + let output = diff(from.as_bytes(), to.as_bytes(), ¶ms); + assert_diff_eq!(output, expected); + + // An insertion before a nearby change gives the two sides different + // line counts; the hunks must still be merged (issue #268). + let from = ["milk", "eggs", "bread", "butter", "apples", ""].join("\n"); + let to = ["milk", "eggs", "yogurt", "bread", "honey", "apples", ""].join("\n"); + let expected = [ + "*** foo\tTIMESTAMP", + "--- bar\tTIMESTAMP", + "***************", + "*** 2,5 ****", + " eggs", + " bread", + "! butter", + " apples", + "--- 2,6 ----", + " eggs", + "+ yogurt", + " bread", + "! honey", + " apples", + "", + ] + .join("\n"); + let output = diff(from.as_bytes(), to.as_bytes(), ¶ms); + assert_diff_eq!(output, expected); + } +} diff --git a/userland/diffutils/src/diff.rs b/userland/diffutils/src/diff.rs new file mode 100644 index 0000000..455023e --- /dev/null +++ b/userland/diffutils/src/diff.rs @@ -0,0 +1,151 @@ +// This file is part of the uutils diffutils package. +// +// For the full copyright and license information, please view the LICENSE-* +// files that was distributed with this source code. + +use crate::params::{parse_params, Format}; +use crate::utils::report_failure_to_read_input_file; +use crate::{context_diff, ed_diff, normal_diff, side_diff, unified_diff}; +use std::env::ArgsOs; +use std::ffi::OsString; +use std::fs; +use std::io::{self, stdout, Read, Write}; +use std::iter::Peekable; +use std::process::{exit, ExitCode}; + +// Exit codes are documented at +// https://www.gnu.org/software/diffutils/manual/html_node/Invoking-diff.html. +// An exit status of 0 means no differences were found, +// 1 means some differences were found, +// and 2 means trouble. +pub fn main(opts: Peekable) -> ExitCode { + let params = parse_params(opts).unwrap_or_else(|error| { + eprintln!("{error}"); + exit(2); + }); + if params.recursive { + return recursive(¶ms).unwrap_or_else(|error| { + eprintln!("diff: {error}"); + ExitCode::from(2) + }); + } + compare(¶ms) +} + +/// Compare directory entries in lexical order, with GNU diff's recursive output. +fn recursive(params: &crate::params::Params) -> io::Result { + let from = std::path::Path::new(¶ms.from); + let to = std::path::Path::new(¶ms.to); + if !from.is_dir() || !to.is_dir() { + if fs::read(from)? != fs::read(to)? { + println!("diff -r {} {}", from.display(), to.display()); + } + return Ok(compare(params)); + } + let mut names = std::collections::BTreeSet::new(); + for dir in [from, to] { + for entry in fs::read_dir(dir)? { names.insert(entry?.file_name()); } + } + let mut status = ExitCode::SUCCESS; + for name in names { + let left = from.join(&name); + let right = to.join(&name); + let result = if !left.exists() || !right.exists() { + let parent = if left.exists() { from } else { to }; + println!("Only in {}: {}", parent.display(), name.to_string_lossy()); + ExitCode::from(1) + } else if left.is_dir() != right.is_dir() { + println!("File {} is a {} while file {} is a {}", left.display(), + if left.is_dir() { "directory" } else { "regular file" }, right.display(), + if right.is_dir() { "directory" } else { "regular file" }); + ExitCode::from(1) + } else { + let mut child = params.clone(); + child.from = left.into_os_string(); + child.to = right.into_os_string(); + recursive(&child)? + }; + if result == ExitCode::from(2) { status = result; } + else if result == ExitCode::from(1) && status == ExitCode::SUCCESS { status = result; } + } + Ok(status) +} + +fn compare(params: &crate::params::Params) -> ExitCode { + // if from and to are the same file, no need to perform any comparison + let maybe_report_identical_files = || { + if params.report_identical_files { + println!( + "Files {} and {} are identical", + params.from.to_string_lossy(), + params.to.to_string_lossy(), + ); + } + }; + if params.from == "-" && params.to == "-" + || same_file::is_same_file(¶ms.from, ¶ms.to).unwrap_or(false) + { + maybe_report_identical_files(); + return ExitCode::SUCCESS; + } + + // read files + fn read_file_contents(filepath: &OsString) -> io::Result> { + if filepath == "-" { + let mut content = Vec::new(); + io::stdin().read_to_end(&mut content).and(Ok(content)) + } else { + fs::read(filepath) + } + } + let mut io_error = false; + let from_content = match read_file_contents(¶ms.from) { + Ok(from_content) => from_content, + Err(e) => { + report_failure_to_read_input_file(¶ms.executable, ¶ms.from, &e); + io_error = true; + vec![] + } + }; + let to_content = match read_file_contents(¶ms.to) { + Ok(to_content) => to_content, + Err(e) => { + report_failure_to_read_input_file(¶ms.executable, ¶ms.to, &e); + io_error = true; + vec![] + } + }; + if io_error { + return ExitCode::from(2); + } + + // run diff + let result: Vec = match params.format { + Format::Normal => normal_diff::diff(&from_content, &to_content, ¶ms), + Format::Unified => unified_diff::diff(&from_content, &to_content, ¶ms), + Format::Context => context_diff::diff(&from_content, &to_content, ¶ms), + Format::Ed => ed_diff::diff(&from_content, &to_content, ¶ms).unwrap_or_else(|error| { + eprintln!("{error}"); + exit(2); + }), + Format::SideBySide => { + let mut output = stdout().lock(); + side_diff::diff(&from_content, &to_content, &mut output, ¶ms) + } + }; + if params.brief && !result.is_empty() { + println!( + "Files {} and {} differ", + params.from.to_string_lossy(), + params.to.to_string_lossy() + ); + } else { + io::stdout().write_all(&result).unwrap(); + } + if result.is_empty() { + maybe_report_identical_files(); + ExitCode::SUCCESS + } else { + ExitCode::from(1) + } +} diff --git a/userland/diffutils/src/ed_diff.rs b/userland/diffutils/src/ed_diff.rs new file mode 100644 index 0000000..81ddfb4 --- /dev/null +++ b/userland/diffutils/src/ed_diff.rs @@ -0,0 +1,452 @@ +// This file is part of the uutils diffutils package. +// +// For the full copyright and license information, please view the LICENSE-* +// files that was distributed with this source code. + +use std::io::Write; + +use crate::params::Params; +use crate::utils::do_write_line; + +#[derive(Debug, PartialEq)] +struct Mismatch { + pub line_number_expected: usize, + pub line_number_actual: usize, + pub expected: Vec>, + pub actual: Vec>, +} + +#[derive(Debug, PartialEq, Eq)] +pub enum DiffError { + MissingNL, +} + +impl std::fmt::Display for DiffError { + fn fmt(&self, f: &mut std::fmt::Formatter) -> Result<(), std::fmt::Error> { + std::fmt::Display::fmt("No newline at end of file", f) + } +} + +impl From for String { + fn from(_: DiffError) -> String { + "No newline at end of file".into() + } +} + +impl Mismatch { + fn new(line_number_expected: usize, line_number_actual: usize) -> Mismatch { + Mismatch { + line_number_expected, + line_number_actual, + expected: Vec::new(), + actual: Vec::new(), + } + } +} + +// Produces a diff between the expected output and actual output. +fn make_diff(expected: &[u8], actual: &[u8], stop_early: bool) -> Result, DiffError> { + let mut line_number_expected = 1; + let mut line_number_actual = 1; + let mut results = Vec::new(); + let mut mismatch = Mismatch::new(line_number_expected, line_number_actual); + + let mut expected_lines: Vec<&[u8]> = expected.split(|&c| c == b'\n').collect(); + let mut actual_lines: Vec<&[u8]> = actual.split(|&c| c == b'\n').collect(); + + debug_assert_eq!(b"".split(|&c| c == b'\n').count(), 1); + // ^ means that underflow here is impossible + let _expected_lines_count = expected_lines.len() - 1; + let _actual_lines_count = actual_lines.len() - 1; + + if expected_lines.last() == Some(&&b""[..]) { + expected_lines.pop(); + } else { + return Err(DiffError::MissingNL); + } + + if actual_lines.last() == Some(&&b""[..]) { + actual_lines.pop(); + } else { + return Err(DiffError::MissingNL); + } + + for result in diff::slice(&expected_lines, &actual_lines) { + match result { + diff::Result::Left(str) => { + if !mismatch.actual.is_empty() { + results.push(mismatch); + mismatch = Mismatch::new(line_number_expected, line_number_actual); + } + mismatch.expected.push(str.to_vec()); + line_number_expected += 1; + } + diff::Result::Right(str) => { + mismatch.actual.push(str.to_vec()); + line_number_actual += 1; + } + diff::Result::Both(_str, _) => { + line_number_expected += 1; + line_number_actual += 1; + if !mismatch.actual.is_empty() || !mismatch.expected.is_empty() { + results.push(mismatch); + mismatch = Mismatch::new(line_number_expected, line_number_actual); + } else { + mismatch.line_number_expected = line_number_expected; + mismatch.line_number_actual = line_number_actual; + } + } + } + if stop_early && !results.is_empty() { + // Optimization: stop analyzing the files as soon as there are any differences + return Ok(results); + } + } + + if !mismatch.actual.is_empty() || !mismatch.expected.is_empty() { + results.push(mismatch); + } + + Ok(results) +} + +pub fn diff(expected: &[u8], actual: &[u8], params: &Params) -> Result, DiffError> { + let mut output = Vec::new(); + let diff_results = make_diff(expected, actual, params.brief)?; + if params.brief && !diff_results.is_empty() { + write!(&mut output, "\0").unwrap(); + return Ok(output); + } + let mut lines_offset = 0; + for result in diff_results { + let line_number_expected: isize = result.line_number_expected as isize + lines_offset; + let _line_number_actual: isize = result.line_number_actual as isize + lines_offset; + let expected_count: isize = result.expected.len() as isize; + let actual_count: isize = result.actual.len() as isize; + match (expected_count, actual_count) { + (0, 0) => unreachable!(), + (0, _) => writeln!(&mut output, "{}a", line_number_expected - 1).unwrap(), + (_, 0) => writeln!( + &mut output, + "{},{}d", + line_number_expected, + expected_count + line_number_expected - 1 + ) + .unwrap(), + (1, _) => writeln!(&mut output, "{line_number_expected}c").unwrap(), + _ => writeln!( + &mut output, + "{},{}c", + line_number_expected, + expected_count + line_number_expected - 1 + ) + .unwrap(), + } + lines_offset += actual_count - expected_count; + if actual_count != 0 { + for actual in &result.actual { + if actual == b"." { + writeln!(&mut output, "..\n.\ns/.//\na").unwrap(); + } else { + do_write_line(&mut output, actual, params.expand_tabs, params.tabsize).unwrap(); + writeln!(&mut output).unwrap(); + } + } + writeln!(&mut output, ".").unwrap(); + } + } + Ok(output) +} + +#[cfg(test)] +mod tests { + use super::*; + use pretty_assertions::assert_eq; + + use crate::utils::testcmds::ED_CMD; + + pub fn diff_w(expected: &[u8], actual: &[u8], filename: &str) -> Result, DiffError> { + let mut output = diff(expected, actual, &Params::default())?; + writeln!(&mut output, "w {filename}").unwrap(); + Ok(output) + } + + #[test] + fn test_basic() { + let from = b"a\n"; + let to = b"b\n"; + let diff = diff(from, to, &Params::default()).unwrap(); + let expected = ["1c", "b", ".", ""].join("\n"); + assert_eq!(diff, expected.as_bytes()); + } + + #[test] + fn test_permutations() { + let target = "target/ed-diff/"; + // test all possible six-line files. + let _ = std::fs::create_dir(target); + for &a in &[0, 1, 2] { + for &b in &[0, 1, 2] { + for &c in &[0, 1, 2] { + for &d in &[0, 1, 2] { + for &e in &[0, 1, 2] { + for &f in &[0, 1, 2] { + use std::fs::File; + use std::io::Write; + let mut alef = Vec::new(); + let mut bet = Vec::new(); + alef.write_all(if a == 0 { b"a\n" } else { b"b\n" }) + .unwrap(); + if a != 2 { + bet.write_all(b"b\n").unwrap(); + } + alef.write_all(if b == 0 { b"c\n" } else { b"d\n" }) + .unwrap(); + if b != 2 { + bet.write_all(b"d\n").unwrap(); + } + alef.write_all(if c == 0 { b"e\n" } else { b"f\n" }) + .unwrap(); + if c != 2 { + bet.write_all(b"f\n").unwrap(); + } + alef.write_all(if d == 0 { b"g\n" } else { b"h\n" }) + .unwrap(); + if d != 2 { + bet.write_all(b"h\n").unwrap(); + } + alef.write_all(if e == 0 { b"i\n" } else { b"j\n" }) + .unwrap(); + if e != 2 { + bet.write_all(b"j\n").unwrap(); + } + alef.write_all(if f == 0 { b"k\n" } else { b"l\n" }) + .unwrap(); + if f != 2 { + bet.write_all(b"l\n").unwrap(); + } + // This test diff is intentionally reversed. + // We want it to turn the alef into bet. + let diff = diff_w(&alef, &bet, &format!("{target}/alef")).unwrap(); + File::create(format!("{target}/ab.ed")) + .unwrap() + .write_all(&diff) + .unwrap(); + let mut fa = File::create(format!("{target}/alef")).unwrap(); + fa.write_all(&alef[..]).unwrap(); + let mut fb = File::create(format!("{target}/bet")).unwrap(); + fb.write_all(&bet[..]).unwrap(); + let _ = fa; + let _ = fb; + #[cfg(not(windows))] // there's no ed on windows + { + let output = ED_CMD + .new() + .arg(format!("{target}/alef")) + .stdin(File::open(format!("{target}/ab.ed")).unwrap()) + .output() + .unwrap(); + assert!(output.status.success(), "{output:?}"); + //println!("{}", String::from_utf8_lossy(&output.stdout)); + //println!("{}", String::from_utf8_lossy(&output.stderr)); + let alef = std::fs::read(format!("{target}/alef")).unwrap(); + assert_eq!(alef, bet); + } + } + } + } + } + } + } + } + + #[test] + fn test_permutations_empty_lines() { + let target = "target/ed-diff/"; + // test all possible six-line files with missing newlines. + let _ = std::fs::create_dir(target); + for &a in &[0, 1, 2] { + for &b in &[0, 1, 2] { + for &c in &[0, 1, 2] { + for &d in &[0, 1, 2] { + for &e in &[0, 1, 2] { + for &f in &[0, 1, 2] { + use std::fs::File; + use std::io::Write; + let mut alef = Vec::new(); + let mut bet = Vec::new(); + alef.write_all(if a == 0 { b"\n" } else { b"b\n" }).unwrap(); + if a != 2 { + bet.write_all(b"b\n").unwrap(); + } + alef.write_all(if b == 0 { b"\n" } else { b"d\n" }).unwrap(); + if b != 2 { + bet.write_all(b"d\n").unwrap(); + } + alef.write_all(if c == 0 { b"\n" } else { b"f\n" }).unwrap(); + if c != 2 { + bet.write_all(b"f\n").unwrap(); + } + alef.write_all(if d == 0 { b"\n" } else { b"h\n" }).unwrap(); + if d != 2 { + bet.write_all(b"h\n").unwrap(); + } + alef.write_all(if e == 0 { b"\n" } else { b"j\n" }).unwrap(); + if e != 2 { + bet.write_all(b"j\n").unwrap(); + } + alef.write_all(if f == 0 { b"\n" } else { b"l\n" }).unwrap(); + if f != 2 { + bet.write_all(b"l\n").unwrap(); + } + // This test diff is intentionally reversed. + // We want it to turn the alef into bet. + let diff = diff_w(&alef, &bet, &format!("{target}/alef_")).unwrap(); + File::create(format!("{target}/ab_.ed")) + .unwrap() + .write_all(&diff) + .unwrap(); + let mut fa = File::create(format!("{target}/alef_")).unwrap(); + fa.write_all(&alef[..]).unwrap(); + let mut fb = File::create(format!("{target}/bet_")).unwrap(); + fb.write_all(&bet[..]).unwrap(); + let _ = fa; + let _ = fb; + #[cfg(not(windows))] // there's no ed on windows + { + let output = ED_CMD + .new() + .arg(format!("{target}/alef_")) + .stdin(File::open(format!("{target}/ab_.ed")).unwrap()) + .output() + .unwrap(); + assert!(output.status.success(), "{output:?}"); + //println!("{}", String::from_utf8_lossy(&output.stdout)); + //println!("{}", String::from_utf8_lossy(&output.stderr)); + let alef = std::fs::read(format!("{target}/alef_")).unwrap(); + assert_eq!(alef, bet); + } + } + } + } + } + } + } + } + + #[test] + fn test_permutations_reverse() { + let target = "target/ed-diff/"; + // test all possible six-line files. + let _ = std::fs::create_dir(target); + for &a in &[0, 1, 2] { + for &b in &[0, 1, 2] { + for &c in &[0, 1, 2] { + for &d in &[0, 1, 2] { + for &e in &[0, 1, 2] { + for &f in &[0, 1, 2] { + use std::fs::File; + use std::io::Write; + let mut alef = Vec::new(); + let mut bet = Vec::new(); + alef.write_all(if a == 0 { b"a\n" } else { b"f\n" }) + .unwrap(); + if a != 2 { + bet.write_all(b"a\n").unwrap(); + } + alef.write_all(if b == 0 { b"b\n" } else { b"e\n" }) + .unwrap(); + if b != 2 { + bet.write_all(b"b\n").unwrap(); + } + alef.write_all(if c == 0 { b"c\n" } else { b"d\n" }) + .unwrap(); + if c != 2 { + bet.write_all(b"c\n").unwrap(); + } + alef.write_all(if d == 0 { b"d\n" } else { b"c\n" }) + .unwrap(); + if d != 2 { + bet.write_all(b"d\n").unwrap(); + } + alef.write_all(if e == 0 { b"e\n" } else { b"b\n" }) + .unwrap(); + if e != 2 { + bet.write_all(b"e\n").unwrap(); + } + alef.write_all(if f == 0 { b"f\n" } else { b"a\n" }) + .unwrap(); + if f != 2 { + bet.write_all(b"f\n").unwrap(); + } + // This test diff is intentionally reversed. + // We want it to turn the alef into bet. + let diff = diff_w(&alef, &bet, &format!("{target}/alefr")).unwrap(); + File::create(format!("{target}/abr.ed")) + .unwrap() + .write_all(&diff) + .unwrap(); + let mut fa = File::create(format!("{target}/alefr")).unwrap(); + fa.write_all(&alef[..]).unwrap(); + let mut fb = File::create(format!("{target}/betr")).unwrap(); + fb.write_all(&bet[..]).unwrap(); + let _ = fa; + let _ = fb; + #[cfg(not(windows))] // there's no ed on windows + { + let output = ED_CMD + .new() + .arg(format!("{target}/alefr")) + .stdin(File::open(format!("{target}/abr.ed")).unwrap()) + .output() + .unwrap(); + assert!(output.status.success(), "{output:?}"); + //println!("{}", String::from_utf8_lossy(&output.stdout)); + //println!("{}", String::from_utf8_lossy(&output.stderr)); + let alef = std::fs::read(format!("{target}/alefr")).unwrap(); + assert_eq!(alef, bet); + } + } + } + } + } + } + } + } + + #[test] + fn test_stop_early() { + let from = ["a", "b", "c", ""].join("\n"); + let to = ["a", "d", "c", ""].join("\n"); + + let diff_full = diff(from.as_bytes(), to.as_bytes(), &Params::default()).unwrap(); + let expected_full = ["2c", "d", ".", ""].join("\n"); + assert_eq!(diff_full, expected_full.as_bytes()); + + let diff_brief = diff( + from.as_bytes(), + to.as_bytes(), + &Params { + brief: true, + ..Default::default() + }, + ) + .unwrap(); + let expected_brief = "\0".as_bytes(); + assert_eq!(diff_brief, expected_brief); + + let nodiff_full = diff(from.as_bytes(), from.as_bytes(), &Params::default()).unwrap(); + assert!(nodiff_full.is_empty()); + + let nodiff_brief = diff( + from.as_bytes(), + from.as_bytes(), + &Params { + brief: true, + ..Default::default() + }, + ) + .unwrap(); + assert!(nodiff_brief.is_empty()); + } +} diff --git a/userland/diffutils/src/lib.rs b/userland/diffutils/src/lib.rs new file mode 100644 index 0000000..342b01c --- /dev/null +++ b/userland/diffutils/src/lib.rs @@ -0,0 +1,16 @@ +pub mod cmp; +pub mod context_diff; +pub mod ed_diff; +pub mod macros; +pub mod normal_diff; +pub mod params; +pub mod side_diff; +pub mod unified_diff; +pub mod utils; + +// Re-export the public functions/types you need +pub use context_diff::diff as context_diff; +pub use ed_diff::diff as ed_diff; +pub use normal_diff::diff as normal_diff; +pub use side_diff::diff as side_by_side_diff; +pub use unified_diff::diff as unified_diff; diff --git a/userland/diffutils/src/macros.rs b/userland/diffutils/src/macros.rs new file mode 100644 index 0000000..90a4eaa --- /dev/null +++ b/userland/diffutils/src/macros.rs @@ -0,0 +1,25 @@ +// asserts equality of the actual diff and expected diff +// considering datetime varitations +// +// It replaces the modification time in the actual diff +// with placeholder "TIMESTAMP" and then asserts the equality +// +// For eg. +// let brief = "*** fruits_old.txt\t2024-03-24 23:43:05.189597645 +0530\n +// --- fruits_new.txt\t2024-03-24 23:35:08.922581904 +0530\n"; +// +// replaced = "*** fruits_old.txt\tTIMESTAMP\n +// --- fruits_new.txt\tTIMESTAMP\n"; +#[macro_export] +macro_rules! assert_diff_eq { + ($actual:expr, $expected:expr) => {{ + use regex::Regex; + use std::str; + + let diff = str::from_utf8(&$actual).unwrap(); + let re = Regex::new(r"\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d+ [+-]\d{4}").unwrap(); + let actual = re.replacen(diff, 2, "TIMESTAMP"); + + assert_eq!(actual, $expected); + }}; +} diff --git a/userland/diffutils/src/main.rs b/userland/diffutils/src/main.rs new file mode 100644 index 0000000..d0adb69 --- /dev/null +++ b/userland/diffutils/src/main.rs @@ -0,0 +1,85 @@ +// This file is part of the uutils diffutils package. +// +// For the full copyright and license information, please view the LICENSE-* +// files that was distributed with this source code. + +use std::{ + env::ArgsOs, + ffi::{OsStr, OsString}, + iter::Peekable, + path::{Path, PathBuf}, + process::ExitCode, +}; + +mod cmp; +mod context_diff; +mod diff; +mod ed_diff; +mod macros; +mod normal_diff; +mod params; +mod side_diff; +mod unified_diff; +mod utils; + +/// # Panics +/// Panics if the binary path cannot be determined +fn binary_path(args: &mut Peekable) -> PathBuf { + match args.peek() { + Some(ref s) if !s.is_empty() => PathBuf::from(s), + _ => std::env::current_exe().unwrap(), + } +} + +/// #Panics +/// Panics if path has no UTF-8 valid name +fn name(binary_path: &Path) -> &OsStr { + binary_path.file_stem().unwrap() +} + +const VERSION: &str = env!("CARGO_PKG_VERSION"); + +fn usage(name: &str) { + println!("{name} {VERSION} (multi-call binary)\n"); + println!("Usage: {name} [function [arguments...]]\n"); + println!("Currently defined functions:\n"); + println!(" cmp, diff\n"); +} + +fn second_arg_error(name: &OsStr) -> ! { + eprintln!("Expected utility name as second argument, got nothing."); + usage(&name.to_string_lossy()); + std::process::exit(0); +} + +fn main() -> ExitCode { + let mut args = std::env::args_os().peekable(); + + let exe_path = binary_path(&mut args); + let exe_name = name(&exe_path); + + let util_name = if exe_name.as_encoded_bytes().ends_with(b"diffutils") { + // Discard the item we peeked. + let _ = args.next(); + + args.peek() + .cloned() + .unwrap_or_else(|| second_arg_error(exe_name)) + } else { + OsString::from(exe_name) + }; + + match util_name.as_encoded_bytes() { + name if name.ends_with(b"diff") => diff::main(args), + name if name.ends_with(b"cmp") => cmp::main(args), + name => { + use std::io::{stderr, Write as _}; + let _ = writeln!( + stderr(), + "{}: utility not supported", + String::from_utf8_lossy(name) + ); + ExitCode::from(2) + } + } +} diff --git a/userland/diffutils/src/normal_diff.rs b/userland/diffutils/src/normal_diff.rs new file mode 100644 index 0000000..652dc84 --- /dev/null +++ b/userland/diffutils/src/normal_diff.rs @@ -0,0 +1,586 @@ +// This file is part of the uutils diffutils package. +// +// For the full copyright and license information, please view the LICENSE-* +// files that was distributed with this source code. + +use std::io::Write; + +use crate::params::Params; +use crate::utils::do_write_line; + +#[derive(Debug, PartialEq)] +struct Mismatch { + pub line_number_expected: usize, + pub line_number_actual: usize, + pub expected: Vec>, + pub actual: Vec>, + pub expected_missing_nl: bool, + pub actual_missing_nl: bool, +} + +impl Mismatch { + fn new(line_number_expected: usize, line_number_actual: usize) -> Mismatch { + Mismatch { + line_number_expected, + line_number_actual, + expected: Vec::new(), + actual: Vec::new(), + expected_missing_nl: false, + actual_missing_nl: false, + } + } +} + +// Produces a diff between the expected output and actual output. +fn make_diff(expected: &[u8], actual: &[u8], stop_early: bool) -> Vec { + let mut line_number_expected = 1; + let mut line_number_actual = 1; + let mut results = Vec::new(); + let mut mismatch = Mismatch::new(line_number_expected, line_number_actual); + + let mut expected_lines: Vec<&[u8]> = expected.split(|&c| c == b'\n').collect(); + let mut actual_lines: Vec<&[u8]> = actual.split(|&c| c == b'\n').collect(); + + debug_assert_eq!(b"".split(|&c| c == b'\n').count(), 1); + // ^ means that underflow here is impossible + let expected_lines_count = expected_lines.len() - 1; + let actual_lines_count = actual_lines.len() - 1; + + if expected_lines.last() == Some(&&b""[..]) { + expected_lines.pop(); + } + + if actual_lines.last() == Some(&&b""[..]) { + actual_lines.pop(); + } + + for result in diff::slice(&expected_lines, &actual_lines) { + match result { + diff::Result::Left(str) => { + if !mismatch.actual.is_empty() && !mismatch.actual_missing_nl { + results.push(mismatch); + mismatch = Mismatch::new(line_number_expected, line_number_actual); + } + mismatch.expected.push(str.to_vec()); + mismatch.expected_missing_nl = line_number_expected > expected_lines_count; + line_number_expected += 1; + } + diff::Result::Right(str) => { + mismatch.actual.push(str.to_vec()); + mismatch.actual_missing_nl = line_number_actual > actual_lines_count; + line_number_actual += 1; + } + diff::Result::Both(str, _) => { + match ( + line_number_expected > expected_lines_count, + line_number_actual > actual_lines_count, + ) { + (true, false) => { + line_number_expected += 1; + line_number_actual += 1; + mismatch.expected.push(str.to_vec()); + mismatch.expected_missing_nl = true; + mismatch.actual.push(str.to_vec()); + } + (false, true) => { + line_number_expected += 1; + line_number_actual += 1; + mismatch.actual.push(str.to_vec()); + mismatch.actual_missing_nl = true; + mismatch.expected.push(str.to_vec()); + } + (true, true) | (false, false) => { + line_number_expected += 1; + line_number_actual += 1; + if !mismatch.actual.is_empty() || !mismatch.expected.is_empty() { + results.push(mismatch); + mismatch = Mismatch::new(line_number_expected, line_number_actual); + } else { + mismatch.line_number_expected = line_number_expected; + mismatch.line_number_actual = line_number_actual; + } + } + } + } + } + if stop_early && !results.is_empty() { + // Optimization: stop analyzing the files as soon as there are any differences + return results; + } + } + + if !mismatch.actual.is_empty() || !mismatch.expected.is_empty() { + results.push(mismatch); + } + + results +} + +#[must_use] +pub fn diff(expected: &[u8], actual: &[u8], params: &Params) -> Vec { + // See https://www.gnu.org/software/diffutils/manual/html_node/Detailed-Normal.html + // for details on the syntax of the normal format. + let mut output = Vec::new(); + let diff_results = make_diff(expected, actual, params.brief); + if params.brief && !diff_results.is_empty() { + write!(&mut output, "\0").unwrap(); + return output; + } + for result in diff_results { + let line_number_expected = result.line_number_expected; + let line_number_actual = result.line_number_actual; + let expected_count = result.expected.len(); + let actual_count = result.actual.len(); + match (expected_count, actual_count) { + (0, 0) => unreachable!(), + (0, _) => writeln!( + // 'a' stands for "Add lines" + &mut output, + "{}a{},{}", + line_number_expected - 1, + line_number_actual, + line_number_actual + actual_count - 1 + ) + .unwrap(), + (_, 0) => writeln!( + // 'd' stands for "Delete lines" + &mut output, + "{},{}d{}", + line_number_expected, + expected_count + line_number_expected - 1, + line_number_actual - 1 + ) + .unwrap(), + (1, 1) => writeln!( + // 'c' stands for "Change lines" + // exactly one line replaced by one line + &mut output, + "{line_number_expected}c{line_number_actual}" + ) + .unwrap(), + (1, _) => writeln!( + // one line replaced by multiple lines + &mut output, + "{}c{},{}", + line_number_expected, + line_number_actual, + actual_count + line_number_actual - 1 + ) + .unwrap(), + (_, 1) => writeln!( + // multiple lines replaced by one line + &mut output, + "{},{}c{}", + line_number_expected, + expected_count + line_number_expected - 1, + line_number_actual + ) + .unwrap(), + _ => writeln!( + // general case: multiple lines replaced by multiple lines + &mut output, + "{},{}c{},{}", + line_number_expected, + expected_count + line_number_expected - 1, + line_number_actual, + actual_count + line_number_actual - 1 + ) + .unwrap(), + } + for expected in &result.expected { + write!(&mut output, "< ").unwrap(); + do_write_line(&mut output, expected, params.expand_tabs, params.tabsize).unwrap(); + writeln!(&mut output).unwrap(); + } + if result.expected_missing_nl { + writeln!(&mut output, r"\ No newline at end of file").unwrap(); + } + if expected_count != 0 && actual_count != 0 { + writeln!(&mut output, "---").unwrap(); + } + for actual in &result.actual { + write!(&mut output, "> ").unwrap(); + do_write_line(&mut output, actual, params.expand_tabs, params.tabsize).unwrap(); + writeln!(&mut output).unwrap(); + } + if result.actual_missing_nl { + writeln!(&mut output, r"\ No newline at end of file").unwrap(); + } + } + output +} + +#[cfg(test)] +mod tests { + use super::*; + use pretty_assertions::assert_eq; + + use crate::utils::testcmds::PATCH_CMD; + + #[test] + fn test_basic() { + let mut a = Vec::new(); + a.write_all(b"a\n").unwrap(); + let mut b = Vec::new(); + b.write_all(b"b\n").unwrap(); + let diff = diff(&a, &b, &Params::default()); + let expected = b"1c1\n< a\n---\n> b\n".to_vec(); + assert_eq!(diff, expected); + } + + #[test] + fn test_permutations() { + let target = "target/normal-diff/"; + // test all possible six-line files. + let _ = std::fs::create_dir(target); + for &a in &[0, 1, 2] { + for &b in &[0, 1, 2] { + for &c in &[0, 1, 2] { + for &d in &[0, 1, 2] { + for &e in &[0, 1, 2] { + for &f in &[0, 1, 2] { + use std::fs::{self, File}; + use std::io::Write; + let mut alef = Vec::new(); + let mut bet = Vec::new(); + alef.write_all(if a == 0 { b"a\n" } else { b"b\n" }) + .unwrap(); + if a != 2 { + bet.write_all(b"b\n").unwrap(); + } + alef.write_all(if b == 0 { b"c\n" } else { b"d\n" }) + .unwrap(); + if b != 2 { + bet.write_all(b"d\n").unwrap(); + } + alef.write_all(if c == 0 { b"e\n" } else { b"f\n" }) + .unwrap(); + if c != 2 { + bet.write_all(b"f\n").unwrap(); + } + alef.write_all(if d == 0 { b"g\n" } else { b"h\n" }) + .unwrap(); + if d != 2 { + bet.write_all(b"h\n").unwrap(); + } + alef.write_all(if e == 0 { b"i\n" } else { b"j\n" }) + .unwrap(); + if e != 2 { + bet.write_all(b"j\n").unwrap(); + } + alef.write_all(if f == 0 { b"k\n" } else { b"l\n" }) + .unwrap(); + if f != 2 { + bet.write_all(b"l\n").unwrap(); + } + // This test diff is intentionally reversed. + // We want it to turn the alef into bet. + let diff = diff(&alef, &bet, &Params::default()); + File::create(format!("{target}/ab.diff")) + .unwrap() + .write_all(&diff) + .unwrap(); + let mut fa = File::create(format!("{target}/alef")).unwrap(); + fa.write_all(&alef[..]).unwrap(); + let mut fb = File::create(format!("{target}/bet")).unwrap(); + fb.write_all(&bet[..]).unwrap(); + let _ = fa; + let _ = fb; + let output = PATCH_CMD + .new() + .arg("-p0") + .arg(format!("{target}/alef")) + .stdin(File::open(format!("{target}/ab.diff")).unwrap()) + .output() + .unwrap(); + assert!(output.status.success(), "{output:?}"); + //println!("{}", String::from_utf8_lossy(&output.stdout)); + //println!("{}", String::from_utf8_lossy(&output.stderr)); + let alef = fs::read(format!("{target}/alef")).unwrap(); + assert_eq!(alef, bet); + } + } + } + } + } + } + } + + #[test] + fn test_permutations_missing_line_ending() { + let target = "target/normal-diff/"; + // test all possible six-line files with missing newlines. + let _ = std::fs::create_dir(target); + for &a in &[0, 1, 2] { + for &b in &[0, 1, 2] { + for &c in &[0, 1, 2] { + for &d in &[0, 1, 2] { + for &e in &[0, 1, 2] { + for &f in &[0, 1, 2] { + for &g in &[0, 1, 2] { + use std::fs::{self, File}; + use std::io::Write; + let mut alef = Vec::new(); + let mut bet = Vec::new(); + alef.write_all(if a == 0 { b"a\n" } else { b"b\n" }) + .unwrap(); + if a != 2 { + bet.write_all(b"b\n").unwrap(); + } + alef.write_all(if b == 0 { b"c\n" } else { b"d\n" }) + .unwrap(); + if b != 2 { + bet.write_all(b"d\n").unwrap(); + } + alef.write_all(if c == 0 { b"e\n" } else { b"f\n" }) + .unwrap(); + if c != 2 { + bet.write_all(b"f\n").unwrap(); + } + alef.write_all(if d == 0 { b"g\n" } else { b"h\n" }) + .unwrap(); + if d != 2 { + bet.write_all(b"h\n").unwrap(); + } + alef.write_all(if e == 0 { b"i\n" } else { b"j\n" }) + .unwrap(); + if e != 2 { + bet.write_all(b"j\n").unwrap(); + } + alef.write_all(if f == 0 { b"k\n" } else { b"l\n" }) + .unwrap(); + if f != 2 { + bet.write_all(b"l\n").unwrap(); + } + match g { + 0 => { + alef.pop(); + } + 1 => { + bet.pop(); + } + 2 => { + alef.pop(); + bet.pop(); + } + _ => unreachable!(), + } + // This test diff is intentionally reversed. + // We want it to turn the alef into bet. + let diff = diff(&alef, &bet, &Params::default()); + File::create(format!("{target}/abn.diff")) + .unwrap() + .write_all(&diff) + .unwrap(); + let mut fa = File::create(format!("{target}/alefn")).unwrap(); + fa.write_all(&alef[..]).unwrap(); + let mut fb = File::create(format!("{target}/betn")).unwrap(); + fb.write_all(&bet[..]).unwrap(); + let _ = fa; + let _ = fb; + let output = PATCH_CMD + .new() + .arg("-p0") + .arg("--normal") + .arg(format!("{target}/alefn")) + .stdin(File::open(format!("{target}/abn.diff")).unwrap()) + .output() + .unwrap(); + assert!(output.status.success(), "{output:?}"); + //println!("{}", String::from_utf8_lossy(&output.stdout)); + //println!("{}", String::from_utf8_lossy(&output.stderr)); + let alef = fs::read(format!("{target}/alefn")).unwrap(); + assert_eq!(alef, bet); + } + } + } + } + } + } + } + } + + #[test] + fn test_permutations_empty_lines() { + let target = "target/normal-diff/"; + // test all possible six-line files with missing newlines. + let _ = std::fs::create_dir(target); + for &a in &[0, 1, 2] { + for &b in &[0, 1, 2] { + for &c in &[0, 1, 2] { + for &d in &[0, 1, 2] { + for &e in &[0, 1, 2] { + for &f in &[0, 1, 2] { + use std::fs::{self, File}; + use std::io::Write; + let mut alef = Vec::new(); + let mut bet = Vec::new(); + alef.write_all(if a == 0 { b"\n" } else { b"b\n" }).unwrap(); + if a != 2 { + bet.write_all(b"b\n").unwrap(); + } + alef.write_all(if b == 0 { b"\n" } else { b"d\n" }).unwrap(); + if b != 2 { + bet.write_all(b"d\n").unwrap(); + } + alef.write_all(if c == 0 { b"\n" } else { b"f\n" }).unwrap(); + if c != 2 { + bet.write_all(b"f\n").unwrap(); + } + alef.write_all(if d == 0 { b"\n" } else { b"h\n" }).unwrap(); + if d != 2 { + bet.write_all(b"h\n").unwrap(); + } + alef.write_all(if e == 0 { b"\n" } else { b"j\n" }).unwrap(); + if e != 2 { + bet.write_all(b"j\n").unwrap(); + } + alef.write_all(if f == 0 { b"\n" } else { b"l\n" }).unwrap(); + if f != 2 { + bet.write_all(b"l\n").unwrap(); + } + // This test diff is intentionally reversed. + // We want it to turn the alef into bet. + let diff = diff(&alef, &bet, &Params::default()); + File::create(format!("{target}/ab_.diff")) + .unwrap() + .write_all(&diff) + .unwrap(); + let mut fa = File::create(format!("{target}/alef_")).unwrap(); + fa.write_all(&alef[..]).unwrap(); + let mut fb = File::create(format!("{target}/bet_")).unwrap(); + fb.write_all(&bet[..]).unwrap(); + let _ = fa; + let _ = fb; + let output = PATCH_CMD + .new() + .arg("-p0") + .arg(format!("{target}/alef_")) + .stdin(File::open(format!("{target}/ab_.diff")).unwrap()) + .output() + .unwrap(); + assert!(output.status.success(), "{output:?}"); + //println!("{}", String::from_utf8_lossy(&output.stdout)); + //println!("{}", String::from_utf8_lossy(&output.stderr)); + let alef = fs::read(format!("{target}/alef_")).unwrap(); + assert_eq!(alef, bet); + } + } + } + } + } + } + } + + #[test] + fn test_permutations_reverse() { + let target = "target/normal-diff/"; + // test all possible six-line files. + let _ = std::fs::create_dir(target); + for &a in &[0, 1, 2] { + for &b in &[0, 1, 2] { + for &c in &[0, 1, 2] { + for &d in &[0, 1, 2] { + for &e in &[0, 1, 2] { + for &f in &[0, 1, 2] { + use std::fs::{self, File}; + use std::io::Write; + let mut alef = Vec::new(); + let mut bet = Vec::new(); + alef.write_all(if a == 0 { b"a\n" } else { b"f\n" }) + .unwrap(); + if a != 2 { + bet.write_all(b"a\n").unwrap(); + } + alef.write_all(if b == 0 { b"b\n" } else { b"e\n" }) + .unwrap(); + if b != 2 { + bet.write_all(b"b\n").unwrap(); + } + alef.write_all(if c == 0 { b"c\n" } else { b"d\n" }) + .unwrap(); + if c != 2 { + bet.write_all(b"c\n").unwrap(); + } + alef.write_all(if d == 0 { b"d\n" } else { b"c\n" }) + .unwrap(); + if d != 2 { + bet.write_all(b"d\n").unwrap(); + } + alef.write_all(if e == 0 { b"e\n" } else { b"b\n" }) + .unwrap(); + if e != 2 { + bet.write_all(b"e\n").unwrap(); + } + alef.write_all(if f == 0 { b"f\n" } else { b"a\n" }) + .unwrap(); + if f != 2 { + bet.write_all(b"f\n").unwrap(); + } + // This test diff is intentionally reversed. + // We want it to turn the alef into bet. + let diff = diff(&alef, &bet, &Params::default()); + File::create(format!("{target}/abr.diff")) + .unwrap() + .write_all(&diff) + .unwrap(); + let mut fa = File::create(format!("{target}/alefr")).unwrap(); + fa.write_all(&alef[..]).unwrap(); + let mut fb = File::create(format!("{target}/betr")).unwrap(); + fb.write_all(&bet[..]).unwrap(); + let _ = fa; + let _ = fb; + let output = PATCH_CMD + .new() + .arg("-p0") + .arg(format!("{target}/alefr")) + .stdin(File::open(format!("{target}/abr.diff")).unwrap()) + .output() + .unwrap(); + assert!(output.status.success(), "{output:?}"); + //println!("{}", String::from_utf8_lossy(&output.stdout)); + //println!("{}", String::from_utf8_lossy(&output.stderr)); + let alef = fs::read(format!("{target}/alefr")).unwrap(); + assert_eq!(alef, bet); + } + } + } + } + } + } + } + + #[test] + fn test_stop_early() { + let from = ["a", "b", "c"].join("\n"); + let to = ["a", "d", "c"].join("\n"); + + let diff_full = diff(from.as_bytes(), to.as_bytes(), &Params::default()); + let expected_full = ["2c2", "< b", "---", "> d", ""].join("\n"); + assert_eq!(diff_full, expected_full.as_bytes()); + + let diff_brief = diff( + from.as_bytes(), + to.as_bytes(), + &Params { + brief: true, + ..Default::default() + }, + ); + let expected_brief = "\0".as_bytes(); + assert_eq!(diff_brief, expected_brief); + + let nodiff_full = diff(from.as_bytes(), from.as_bytes(), &Params::default()); + assert!(nodiff_full.is_empty()); + + let nodiff_brief = diff( + from.as_bytes(), + from.as_bytes(), + &Params { + brief: true, + ..Default::default() + }, + ); + assert!(nodiff_brief.is_empty()); + } +} diff --git a/userland/diffutils/src/params.rs b/userland/diffutils/src/params.rs new file mode 100644 index 0000000..314b32f --- /dev/null +++ b/userland/diffutils/src/params.rs @@ -0,0 +1,975 @@ +use std::ffi::OsString; +use std::iter::Peekable; +use std::path::PathBuf; + +use regex::Regex; + +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] +pub enum Format { + #[default] + Normal, + Unified, + Context, + Ed, + SideBySide, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct Params { + pub executable: OsString, + pub from: OsString, + pub to: OsString, + pub format: Format, + pub context_count: usize, + pub report_identical_files: bool, + pub brief: bool, + pub recursive: bool, + pub expand_tabs: bool, + pub tabsize: usize, + pub width: usize, +} + +impl Default for Params { + fn default() -> Self { + Self { + executable: OsString::default(), + from: OsString::default(), + to: OsString::default(), + format: Format::default(), + context_count: 3, + report_identical_files: false, + brief: false, + recursive: false, + expand_tabs: false, + tabsize: 8, + width: 130, + } + } +} + +pub fn parse_params>(mut opts: Peekable) -> Result { + // parse CLI + + let Some(executable) = opts.next() else { + return Err("Usage: ".to_string()); + }; + let mut params = Params { + executable, + ..Default::default() + }; + let mut from = None; + let mut to = None; + let mut format = None; + let mut context = None; + let tabsize_re = Regex::new(r"^--tabsize=(?\d+)$").unwrap(); + let width_re = Regex::new(r"^--width=(?P\d+)$").unwrap(); + while let Some(param) = opts.next() { + let next_param = opts.peek(); + if param == "--" { + break; + } + if param == "-" { + if from.is_none() { + from = Some(param); + } else if to.is_none() { + to = Some(param); + } else { + return Err(format!( + "Usage: {} ", + params.executable.to_string_lossy() + )); + } + continue; + } + if param == "-r" || param == "--recursive" { + params.recursive = true; + continue; + } + if param == "-s" || param == "--report-identical-files" { + params.report_identical_files = true; + continue; + } + if param == "-q" || param == "--brief" { + params.brief = true; + continue; + } + if param == "-t" || param == "--expand-tabs" { + params.expand_tabs = true; + continue; + } + if param == "--normal" { + if format.is_some() && format != Some(Format::Normal) { + return Err("Conflicting output style options".to_string()); + } + format = Some(Format::Normal); + continue; + } + if param == "-e" || param == "--ed" { + if format.is_some() && format != Some(Format::Ed) { + return Err("Conflicting output style options".to_string()); + } + format = Some(Format::Ed); + continue; + } + if param == "-y" || param == "--side-by-side" { + if format.is_some() && format != Some(Format::SideBySide) { + return Err("Conflicting output style option".to_string()); + } + format = Some(Format::SideBySide); + continue; + } + if width_re.is_match(param.to_string_lossy().as_ref()) { + let param = param.into_string().unwrap(); + let width_str: &str = width_re + .captures(param.as_str()) + .unwrap() + .name("long") + .unwrap() + .as_str(); + + params.width = match width_str.parse::() { + Ok(num) => { + if num == 0 { + return Err("invalid width «0»".to_string()); + } + + num + } + Err(_) => return Err(format!("invalid width «{width_str}»")), + }; + continue; + } + if tabsize_re.is_match(param.to_string_lossy().as_ref()) { + // Because param matches the regular expression, + // it is safe to assume it is valid UTF-8. + let param = param.into_string().unwrap(); + let tabsize_str = tabsize_re + .captures(param.as_str()) + .unwrap() + .name("num") + .unwrap() + .as_str(); + params.tabsize = match tabsize_str.parse::() { + Ok(num) => { + if num == 0 { + return Err("invalid tabsize «0»".to_string()); + } + + num + } + Err(_) => return Err(format!("invalid tabsize «{tabsize_str}»")), + }; + + continue; + } + match match_context_diff_params(¶m, next_param, format) { + Ok(DiffStyleMatch { + is_match, + context_count, + next_param_consumed, + }) => { + if is_match { + format = Some(Format::Context); + if context_count.is_some() { + context = context_count; + } + if next_param_consumed { + opts.next(); + } + continue; + } + } + Err(error) => return Err(error), + } + match match_unified_diff_params(¶m, next_param, format) { + Ok(DiffStyleMatch { + is_match, + context_count, + next_param_consumed, + }) => { + if is_match { + format = Some(Format::Unified); + if context_count.is_some() { + context = context_count; + } + if next_param_consumed { + opts.next(); + } + continue; + } + } + Err(error) => return Err(error), + } + if param.to_string_lossy().starts_with('-') { + return Err(format!("unrecognized option '{}'", param.to_string_lossy())); + } + if from.is_none() { + from = Some(param); + } else if to.is_none() { + to = Some(param); + } else { + return Err(format!( + "Usage: {} ", + params.executable.to_string_lossy() + )); + } + } + params.from = if let Some(from) = from { + from + } else if let Some(param) = opts.next() { + param + } else { + return Err(format!( + "Usage: {} ", + params.executable.to_string_lossy() + )); + }; + params.to = if let Some(to) = to { + to + } else if let Some(param) = opts.next() { + param + } else { + return Err(format!( + "Usage: {} ", + params.executable.to_string_lossy() + )); + }; + + // diff DIRECTORY FILE => diff DIRECTORY/FILE FILE + // diff FILE DIRECTORY => diff FILE DIRECTORY/FILE + let mut from_path: PathBuf = PathBuf::from(¶ms.from); + let mut to_path: PathBuf = PathBuf::from(¶ms.to); + + if from_path.is_dir() && to_path.is_file() { + from_path.push(to_path.file_name().unwrap()); + params.from = from_path.into_os_string(); + } else if from_path.is_file() && to_path.is_dir() { + to_path.push(from_path.file_name().unwrap()); + params.to = to_path.into_os_string(); + } + + params.format = format.unwrap_or(Format::default()); + if let Some(context_count) = context { + params.context_count = context_count; + } + Ok(params) +} + +struct DiffStyleMatch { + is_match: bool, + context_count: Option, + next_param_consumed: bool, +} + +fn match_context_diff_params( + param: &OsString, + next_param: Option<&OsString>, + format: Option, +) -> Result { + const CONTEXT_RE: &str = r"^(-[cC](?\d*)|--context(=(?\d*))?|-(?\d+)c)$"; + let regex = Regex::new(CONTEXT_RE).unwrap(); + let is_match = regex.is_match(param.to_string_lossy().as_ref()); + let mut context_count = None; + let mut next_param_consumed = false; + if is_match { + if format.is_some() && format != Some(Format::Context) { + return Err("Conflicting output style options".to_string()); + } + let captures = regex.captures(param.to_str().unwrap()).unwrap(); + let num = captures + .name("num1") + .or(captures.name("num2")) + .or(captures.name("num3")); + if let Some(numvalue) = num { + if !numvalue.as_str().is_empty() { + context_count = Some(numvalue.as_str().parse::().unwrap()); + } + } + if param == "-C" { + if let Some(p) = next_param { + let size_str = p.to_string_lossy(); + match size_str.parse::() { + Ok(context_size) => { + context_count = Some(context_size); + next_param_consumed = true; + } + Err(_) => return Err(format!("invalid context length '{size_str}'")), + } + } + } + } + Ok(DiffStyleMatch { + is_match, + context_count, + next_param_consumed, + }) +} + +fn match_unified_diff_params( + param: &OsString, + next_param: Option<&OsString>, + format: Option, +) -> Result { + const UNIFIED_RE: &str = r"^(-[uU](?\d*)|--unified(=(?\d*))?|-(?\d+)u)$"; + let regex = Regex::new(UNIFIED_RE).unwrap(); + let is_match = regex.is_match(param.to_string_lossy().as_ref()); + let mut context_count = None; + let mut next_param_consumed = false; + if is_match { + if format.is_some() && format != Some(Format::Unified) { + return Err("Conflicting output style options".to_string()); + } + let captures = regex.captures(param.to_str().unwrap()).unwrap(); + let num = captures + .name("num1") + .or(captures.name("num2")) + .or(captures.name("num3")); + if let Some(numvalue) = num { + if !numvalue.as_str().is_empty() { + context_count = Some(numvalue.as_str().parse::().unwrap()); + } + } + if param == "-U" { + if let Some(p) = next_param { + let size_str = p.to_string_lossy(); + match size_str.parse::() { + Ok(context_size) => { + context_count = Some(context_size); + next_param_consumed = true; + } + Err(_) => return Err(format!("invalid context length '{size_str}'")), + } + } + } + } + Ok(DiffStyleMatch { + is_match, + context_count, + next_param_consumed, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + fn os(s: &str) -> OsString { + OsString::from(s) + } + #[test] + fn basics() { + assert_eq!( + Ok(Params { + executable: os("diff"), + from: os("foo"), + to: os("bar"), + ..Default::default() + }), + parse_params( + [os("diff"), os("foo"), os("bar")] + .iter() + .cloned() + .peekable() + ) + ); + assert_eq!( + Ok(Params { + executable: os("diff"), + from: os("foo"), + to: os("bar"), + ..Default::default() + }), + parse_params( + [os("diff"), os("--normal"), os("foo"), os("bar")] + .iter() + .cloned() + .peekable() + ) + ); + } + #[test] + fn basics_ed() { + for arg in ["-e", "--ed"] { + assert_eq!( + Ok(Params { + executable: os("diff"), + from: os("foo"), + to: os("bar"), + format: Format::Ed, + ..Default::default() + }), + parse_params( + [os("diff"), os(arg), os("foo"), os("bar")] + .iter() + .cloned() + .peekable() + ) + ); + } + } + #[test] + fn context_valid() { + for args in [vec!["-c"], vec!["--context"], vec!["--context="]] { + let mut params = vec!["diff"]; + params.extend(args); + params.extend(["foo", "bar"]); + assert_eq!( + Ok(Params { + executable: os("diff"), + from: os("foo"), + to: os("bar"), + format: Format::Context, + ..Default::default() + }), + parse_params(params.iter().map(|x| os(x)).peekable()) + ); + } + for args in [ + vec!["-c42"], + vec!["-C42"], + vec!["-C", "42"], + vec!["--context=42"], + vec!["-42c"], + ] { + let mut params = vec!["diff"]; + params.extend(args); + params.extend(["foo", "bar"]); + assert_eq!( + Ok(Params { + executable: os("diff"), + from: os("foo"), + to: os("bar"), + format: Format::Context, + context_count: 42, + ..Default::default() + }), + parse_params(params.iter().map(|x| os(x)).peekable()) + ); + } + } + #[test] + fn context_invalid() { + for args in [ + vec!["-c", "42"], + vec!["-c=42"], + vec!["-c="], + vec!["-C"], + vec!["-C=42"], + vec!["-C="], + vec!["--context42"], + vec!["--context", "42"], + vec!["-42C"], + ] { + let mut params = vec!["diff"]; + params.extend(args); + params.extend(["foo", "bar"]); + assert!(parse_params(params.iter().map(|x| os(x)).peekable()).is_err()); + } + } + #[test] + fn unified_valid() { + for args in [vec!["-u"], vec!["--unified"], vec!["--unified="]] { + let mut params = vec!["diff"]; + params.extend(args); + params.extend(["foo", "bar"]); + assert_eq!( + Ok(Params { + executable: os("diff"), + from: os("foo"), + to: os("bar"), + format: Format::Unified, + ..Default::default() + }), + parse_params(params.iter().map(|x| os(x)).peekable()) + ); + } + for args in [ + vec!["-u42"], + vec!["-U42"], + vec!["-U", "42"], + vec!["--unified=42"], + vec!["-42u"], + ] { + let mut params = vec!["diff"]; + params.extend(args); + params.extend(["foo", "bar"]); + assert_eq!( + Ok(Params { + executable: os("diff"), + from: os("foo"), + to: os("bar"), + format: Format::Unified, + context_count: 42, + ..Default::default() + }), + parse_params(params.iter().map(|x| os(x)).peekable()) + ); + } + } + #[test] + fn unified_invalid() { + for args in [ + vec!["-u", "42"], + vec!["-u=42"], + vec!["-u="], + vec!["-U"], + vec!["-U=42"], + vec!["-U="], + vec!["--unified42"], + vec!["--unified", "42"], + vec!["-42U"], + ] { + let mut params = vec!["diff"]; + params.extend(args); + params.extend(["foo", "bar"]); + assert!(parse_params(params.iter().map(|x| os(x)).peekable()).is_err()); + } + } + #[test] + fn context_count() { + assert_eq!( + Ok(Params { + executable: os("diff"), + from: os("foo"), + to: os("bar"), + format: Format::Unified, + context_count: 54, + ..Default::default() + }), + parse_params( + [os("diff"), os("-u54"), os("foo"), os("bar")] + .iter() + .cloned() + .peekable() + ) + ); + assert_eq!( + Ok(Params { + executable: os("diff"), + from: os("foo"), + to: os("bar"), + format: Format::Unified, + context_count: 54, + ..Default::default() + }), + parse_params( + [os("diff"), os("-U54"), os("foo"), os("bar")] + .iter() + .cloned() + .peekable() + ) + ); + assert_eq!( + Ok(Params { + executable: os("diff"), + from: os("foo"), + to: os("bar"), + format: Format::Unified, + context_count: 54, + ..Default::default() + }), + parse_params( + [os("diff"), os("-U"), os("54"), os("foo"), os("bar")] + .iter() + .cloned() + .peekable() + ) + ); + assert_eq!( + Ok(Params { + executable: os("diff"), + from: os("foo"), + to: os("bar"), + format: Format::Context, + context_count: 54, + ..Default::default() + }), + parse_params( + [os("diff"), os("-c54"), os("foo"), os("bar")] + .iter() + .cloned() + .peekable() + ) + ); + } + #[test] + fn report_identical_files() { + assert_eq!( + Ok(Params { + executable: os("diff"), + from: os("foo"), + to: os("bar"), + ..Default::default() + }), + parse_params( + [os("diff"), os("foo"), os("bar")] + .iter() + .cloned() + .peekable() + ) + ); + assert_eq!( + Ok(Params { + executable: os("diff"), + from: os("foo"), + to: os("bar"), + report_identical_files: true, + ..Default::default() + }), + parse_params( + [os("diff"), os("-s"), os("foo"), os("bar")] + .iter() + .cloned() + .peekable() + ) + ); + assert_eq!( + Ok(Params { + executable: os("diff"), + from: os("foo"), + to: os("bar"), + report_identical_files: true, + ..Default::default() + }), + parse_params( + [ + os("diff"), + os("--report-identical-files"), + os("foo"), + os("bar"), + ] + .iter() + .cloned() + .peekable() + ) + ); + } + #[test] + fn brief() { + assert_eq!( + Ok(Params { + executable: os("diff"), + from: os("foo"), + to: os("bar"), + ..Default::default() + }), + parse_params( + [os("diff"), os("foo"), os("bar")] + .iter() + .cloned() + .peekable() + ) + ); + assert_eq!( + Ok(Params { + executable: os("diff"), + from: os("foo"), + to: os("bar"), + brief: true, + ..Default::default() + }), + parse_params( + [os("diff"), os("-q"), os("foo"), os("bar")] + .iter() + .cloned() + .peekable() + ) + ); + assert_eq!( + Ok(Params { + executable: os("diff"), + from: os("foo"), + to: os("bar"), + brief: true, + ..Default::default() + }), + parse_params( + [os("diff"), os("--brief"), os("foo"), os("bar"),] + .iter() + .cloned() + .peekable() + ) + ); + } + #[test] + fn expand_tabs() { + assert_eq!( + Ok(Params { + executable: os("diff"), + from: os("foo"), + to: os("bar"), + ..Default::default() + }), + parse_params( + [os("diff"), os("foo"), os("bar")] + .iter() + .cloned() + .peekable() + ) + ); + for option in ["-t", "--expand-tabs"] { + assert_eq!( + Ok(Params { + executable: os("diff"), + from: os("foo"), + to: os("bar"), + expand_tabs: true, + ..Default::default() + }), + parse_params( + [os("diff"), os(option), os("foo"), os("bar")] + .iter() + .cloned() + .peekable() + ) + ); + } + } + #[test] + fn tabsize() { + assert_eq!( + Ok(Params { + executable: os("diff"), + from: os("foo"), + to: os("bar"), + ..Default::default() + }), + parse_params( + [os("diff"), os("foo"), os("bar")] + .iter() + .cloned() + .peekable() + ) + ); + assert_eq!( + Ok(Params { + executable: os("diff"), + from: os("foo"), + to: os("bar"), + tabsize: 1, + ..Default::default() + }), + parse_params( + [os("diff"), os("--tabsize=1"), os("foo"), os("bar")] + .iter() + .cloned() + .peekable() + ) + ); + assert_eq!( + Ok(Params { + executable: os("diff"), + from: os("foo"), + to: os("bar"), + tabsize: 42, + ..Default::default() + }), + parse_params( + [os("diff"), os("--tabsize=42"), os("foo"), os("bar")] + .iter() + .cloned() + .peekable() + ) + ); + assert!(parse_params( + [os("diff"), os("--tabsize"), os("foo"), os("bar")] + .iter() + .cloned() + .peekable() + ) + .is_err()); + assert!(parse_params( + [os("diff"), os("--tabsize="), os("foo"), os("bar")] + .iter() + .cloned() + .peekable() + ) + .is_err()); + assert!(parse_params( + [os("diff"), os("--tabsize=r2"), os("foo"), os("bar")] + .iter() + .cloned() + .peekable() + ) + .is_err()); + assert!(parse_params( + [os("diff"), os("--tabsize=-1"), os("foo"), os("bar")] + .iter() + .cloned() + .peekable() + ) + .is_err()); + assert!(parse_params( + [os("diff"), os("--tabsize=r2"), os("foo"), os("bar")] + .iter() + .cloned() + .peekable() + ) + .is_err()); + assert!(parse_params( + [ + os("diff"), + os("--tabsize=92233720368547758088"), + os("foo"), + os("bar") + ] + .iter() + .cloned() + .peekable() + ) + .is_err()); + } + #[test] + fn width() { + assert_eq!( + Ok(Params { + executable: os("diff"), + from: os("foo"), + to: os("bar"), + width: 100, + ..Default::default() + }), + parse_params( + [os("diff"), os("--width=100"), os("foo"), os("bar")] + .iter() + .cloned() + .peekable() + ) + ); + } + #[test] + fn width_suffix_is_an_operand() { + assert_eq!( + Ok(Params { + executable: os("diff"), + from: os("xyz--width=5"), + to: os("foo"), + ..Default::default() + }), + parse_params( + [os("diff"), os("xyz--width=5"), os("foo")] + .iter() + .cloned() + .peekable() + ) + ); + } + #[cfg(unix)] + #[test] + fn width_non_utf8_is_not_an_option() { + use std::os::unix::ffi::OsStringExt; + // used to panic in into_string().unwrap() + let bad = OsString::from_vec(b"\xff--width=5".to_vec()); + let params = parse_params( + [os("diff"), bad.clone(), os("foo")] + .iter() + .cloned() + .peekable(), + ) + .unwrap(); + assert_eq!(params.from, bad); + } + #[test] + fn double_dash() { + assert_eq!( + Ok(Params { + executable: os("diff"), + from: os("-g"), + to: os("-h"), + ..Default::default() + }), + parse_params( + [os("diff"), os("--"), os("-g"), os("-h")] + .iter() + .cloned() + .peekable() + ) + ); + } + #[test] + fn default_to_stdin() { + assert_eq!( + Ok(Params { + executable: os("diff"), + from: os("foo"), + to: os("-"), + ..Default::default() + }), + parse_params([os("diff"), os("foo"), os("-")].iter().cloned().peekable()) + ); + assert_eq!( + Ok(Params { + executable: os("diff"), + from: os("-"), + to: os("bar"), + ..Default::default() + }), + parse_params([os("diff"), os("-"), os("bar")].iter().cloned().peekable()) + ); + assert_eq!( + Ok(Params { + executable: os("diff"), + from: os("-"), + to: os("-"), + ..Default::default() + }), + parse_params([os("diff"), os("-"), os("-")].iter().cloned().peekable()) + ); + assert!(parse_params( + [os("diff"), os("foo"), os("bar"), os("-")] + .iter() + .cloned() + .peekable() + ) + .is_err()); + assert!(parse_params( + [os("diff"), os("-"), os("-"), os("-")] + .iter() + .cloned() + .peekable() + ) + .is_err()); + } + #[test] + fn missing_arguments() { + assert!(parse_params([os("diff")].iter().cloned().peekable()).is_err()); + assert!(parse_params([os("diff"), os("foo")].iter().cloned().peekable()).is_err()); + } + #[test] + fn unknown_argument() { + assert!(parse_params( + [os("diff"), os("-g"), os("foo"), os("bar")] + .iter() + .cloned() + .peekable() + ) + .is_err()); + assert!( + parse_params([os("diff"), os("-g"), os("bar")].iter().cloned().peekable()).is_err() + ); + assert!(parse_params([os("diff"), os("-g")].iter().cloned().peekable()).is_err()); + } + #[test] + fn empty() { + assert!(parse_params([].iter().cloned().peekable()).is_err()); + } + #[test] + fn conflicting_output_styles() { + for (arg1, arg2) in [ + ("-u", "-c"), + ("-u", "-e"), + ("-c", "-u"), + ("-c", "-U42"), + ("-u", "--normal"), + ("--normal", "-e"), + ("--context", "--normal"), + ] { + assert!(parse_params( + [os("diff"), os(arg1), os(arg2), os("foo"), os("bar")] + .iter() + .cloned() + .peekable() + ) + .is_err()); + } + } +} diff --git a/userland/diffutils/src/side_diff.rs b/userland/diffutils/src/side_diff.rs new file mode 100644 index 0000000..1128e2e --- /dev/null +++ b/userland/diffutils/src/side_diff.rs @@ -0,0 +1,1285 @@ +// This file is part of the uutils diffutils package. +// +// For the full copyright and license information, please view the LICENSE-* +// files that was distributed with this source code. + +use core::cmp::{max, min}; +use diff::Result; +use std::{io::Write, vec}; +use unicode_width::UnicodeWidthStr; + +use crate::params::Params; + +const GUTTER_WIDTH_MIN: usize = 3; + +struct CharIter<'a> { + current: &'a [u8], +} + +struct Config { + sdiff_half_width: usize, + sdiff_column_two_offset: usize, + tab_size: usize, + expanded: bool, + separator_pos: usize, +} + +impl<'a> From<&'a [u8]> for CharIter<'a> { + fn from(value: &'a [u8]) -> Self { + CharIter { current: value } + } +} + +impl<'a> Iterator for CharIter<'a> { + // (bytes for the next char, visible width) + type Item = (&'a [u8], usize); + + fn next(&mut self) -> Option { + let max = self.current.len().min(4); + + // We reached the end. + if max == 0 { + return None; + } + + // Try to find the next utf-8 character, if present in the next 4 bytes. + let mut index = 1; + let mut view = &self.current[..index]; + let mut char = str::from_utf8(view); + while char.is_err() { + index += 1; + if index > max { + break; + } + view = &self.current[..index]; + char = str::from_utf8(view) + } + + match char { + Ok(c) => { + self.current = self + .current + .get(view.len()..) + .unwrap_or(&self.current[0..0]); + Some((view, UnicodeWidthStr::width(c))) + } + Err(_) => { + // We did not find an utf-8 char within the next 4 bytes, return the single byte. + self.current = &self.current[1..]; + Some((&view[..1], 1)) + } + } + } +} + +impl Config { + pub fn new(full_width: usize, tab_size: usize, expanded: bool) -> Self { + // diff uses this calculation to calculate the size of a half line + // based on the options passed (like -w, -t, etc.). It's actually + // pretty useless, because we (actually) don't have any size modifiers + // that can change this, however I just want to leave the calculate + // here, since it's not very clear and may cause some confusion + + let w = full_width as isize; + let t = tab_size as isize; + let t_plus_g = t + GUTTER_WIDTH_MIN as isize; + let unaligned_off = (w >> 1) + (t_plus_g >> 1) + (w & t_plus_g & 1); + let off = unaligned_off - unaligned_off % t; + let hw = max(0, min(off - GUTTER_WIDTH_MIN as isize, w - off)) as usize; + let c2o = if hw != 0 { off as usize } else { w as usize }; + + Self { + expanded, + sdiff_column_two_offset: c2o, + tab_size, + sdiff_half_width: hw, + separator_pos: ((hw + c2o - 1) >> 1), + } + } +} + +fn format_tabs_and_spaces( + from: usize, + to: usize, + config: &Config, + buf: &mut T, +) -> std::io::Result<()> { + let expanded = config.expanded; + let tab_size = config.tab_size; + let mut current = from; + + if current > to { + return Ok(()); + } + + if expanded { + while current < to { + buf.write_all(b" ")?; + current += 1; + } + return Ok(()); + } + + while current + (tab_size - current % tab_size) <= to { + let next_tab = current + (tab_size - current % tab_size); + buf.write_all(b"\t")?; + current = next_tab; + } + + while current < to { + buf.write_all(b" ")?; + current += 1; + } + + Ok(()) +} + +fn process_half_line( + s: &[u8], + max_width: usize, + is_right: bool, + white_space_gutter: bool, + config: &Config, + buf: &mut T, +) -> std::io::Result<()> { + if s.is_empty() { + if !is_right { + format_tabs_and_spaces( + 0, + max_width + + if white_space_gutter { + GUTTER_WIDTH_MIN + } else { + 1 + }, + config, + buf, + )?; + } + + return Ok(()); + } + + if max_width > config.sdiff_half_width { + return Ok(()); + } + + if max_width > config.sdiff_column_two_offset && !is_right { + return Ok(()); + } + + let expanded = config.expanded; + let tab_size = config.tab_size; + let sdiff_column_two_offset = config.sdiff_column_two_offset; + let mut current_width = 0; + let iter = CharIter::from(s); + + // the encoding will probably be compatible with utf8, so we can take advantage + // of that to get the size of the columns and iterate without breaking the encoding of anything. + // It seems like a good trade, since there is still a fallback in case it is not utf8. + // But I think it would be better if we used some lib that would allow us to handle this + // in the best way possible, in order to avoid overhead (currently 2 for loops are needed). + // There is a library called mcel (mcel.h) that is used in GNU diff, but the documentation + // about it is very scarce, nor is its use documented on the internet. In fact, from my + // research I didn't even find any information about it in the GNU lib's own documentation. + + for c in iter { + let (char, c_width) = c; + + if current_width + c_width > max_width { + break; + } + + match char { + b"\t" => { + if expanded && (current_width + tab_size - (current_width % tab_size)) <= max_width + { + let mut spaces = tab_size - (current_width % tab_size); + while spaces > 0 { + buf.write_all(b" ")?; + current_width += 1; + spaces -= 1; + } + } else if current_width + tab_size - (current_width % tab_size) <= max_width { + buf.write_all(b"\t")?; + current_width += tab_size - (current_width % tab_size); + } + } + b"\n" => { + break; + } + b"\r" => { + buf.write_all(b"\r")?; + format_tabs_and_spaces(0, sdiff_column_two_offset, config, buf)?; + current_width = 0; + } + b"\0" | b"\x07" | b"\x0C" | b"\x0B" => { + buf.write_all(char)?; + } + _ => { + buf.write_all(char)?; + current_width += c_width; + } + } + } + + // gnu sdiff do not tabulate the hole empty right line, instead, just keep the line empty + if !is_right { + // we always sum + 1 or + GUTTER_WIDTH_MIN cause we want to expand + // up to the third column of the gutter column if the gutter is gutter white space, + // otherwise we can expand to only the first column of the gutter middle column, cause + // the next is the sep char + format_tabs_and_spaces( + current_width, + max_width + + if white_space_gutter { + GUTTER_WIDTH_MIN + } else { + 1 + }, + config, + buf, + )?; + } + + Ok(()) +} + +fn push_output( + left_ln: &[u8], + right_ln: &[u8], + symbol: u8, + output: &mut T, + config: &Config, +) -> std::io::Result<()> { + if left_ln.is_empty() && right_ln.is_empty() { + writeln!(output)?; + return Ok(()); + } + + let white_space_gutter = symbol == b' '; + let half_width = config.sdiff_half_width; + let column_two_offset = config.sdiff_column_two_offset; + let separator_pos = config.separator_pos; + let put_new_line = true; // should be false when | is allowed + + // this involves a lot of the '|' mark, however, as it is not active, + // it is better to deactivate it as it introduces visual bug if + // the line is empty. + // if !left_ln.is_empty() { + // put_new_line = put_new_line || (left_ln.last() == Some(&b'\n')); + // } + // if !right_ln.is_empty() { + // put_new_line = put_new_line || (right_ln.last() == Some(&b'\n')); + // } + + process_half_line( + left_ln, + half_width, + false, + white_space_gutter, + config, + output, + )?; + if symbol != b' ' { + // the diff always want to put all tabs possible in the usable are, + // even in the middle space between the gutters if possible. + + // Pad to the middle of the gutter so the marker doesn't drift on wider gutters (#269). + format_tabs_and_spaces(half_width + 1, separator_pos, config, output)?; + output.write_all(&[symbol])?; + if !right_ln.is_empty() { + format_tabs_and_spaces(separator_pos + 1, column_two_offset, config, output)?; + } + } + process_half_line( + right_ln, + half_width, + true, + white_space_gutter, + config, + output, + )?; + + if put_new_line { + writeln!(output)?; + } + + Ok(()) +} + +pub fn diff( + from_file: &[u8], + to_file: &[u8], + output: &mut T, + params: &Params, +) -> Vec { + // ^ The left file ^ The right file + + let mut left_lines: Vec<&[u8]> = from_file.split_inclusive(|&c| c == b'\n').collect(); + let mut right_lines: Vec<&[u8]> = to_file.split_inclusive(|&c| c == b'\n').collect(); + let config = Config::new(params.width, params.tabsize, params.expand_tabs); + + if left_lines.last() == Some(&&b""[..]) { + left_lines.pop(); + } + + if right_lines.last() == Some(&&b""[..]) { + right_lines.pop(); + } + + /* + DISCLAIMER: + Currently the diff engine does not produce results like the diff engine used in GNU diff, + so some results may be inaccurate. For example, the line difference marker "|", according + to the GNU documentation, appears when the same lines (only the actual line, although the + relative line may change the result, so occasionally '|' markers appear with the same lines) + are different but exist in both files. In the current solution the same result cannot be + obtained because the diff engine does not return Both if both exist but are different, + but instead returns a Left and a Right for each one, implying that two lines were added + and deleted. Furthermore, the GNU diff program apparently stores some internal state + (this internal state is just a note about how the diff engine works) about the lines. + For example, an added or removed line directly counts in the line query of the original + lines to be printed in the output. Because of this imbalance caused by additions and + deletions, the characters ( and ) are introduced. They basically represent lines without + context, which have lost their pair in the other file due to additions or deletions. Anyway, + my goal with this disclaimer is to warn that for some reason, whether it's the diff engine's + inability to determine and predict/precalculate the result of GNU's sdiff, with this software it's + not possible to reproduce results that are 100% faithful to GNU's, however, the basic premise + e of side diff of showing added and removed lines and creating edit scripts is totally possible. + More studies are needed to cover GNU diff side by side with 100% accuracy, which is one of + the goals of this project : ) + */ + for result in diff::slice(&left_lines, &right_lines) { + match result { + Result::Left(left_ln) => push_output(left_ln, b"", b'<', output, &config).unwrap(), + Result::Right(right_ln) => push_output(b"", right_ln, b'>', output, &config).unwrap(), + Result::Both(left_ln, right_ln) => { + push_output(left_ln, right_ln, b' ', output, &config).unwrap() + } + } + } + + vec![] +} + +#[cfg(test)] +mod tests { + const DEF_TAB_SIZE: usize = 4; + + use super::*; + + mod format_tabs_and_spaces { + use super::*; + + const CONFIG_E_T: Config = Config { + sdiff_half_width: 60, + tab_size: DEF_TAB_SIZE, + expanded: true, + sdiff_column_two_offset: 0, + separator_pos: 0, + }; + + const CONFIG_E_F: Config = Config { + sdiff_half_width: 60, + tab_size: DEF_TAB_SIZE, + expanded: false, + sdiff_column_two_offset: 0, + separator_pos: 0, + }; + + #[test] + fn test_format_tabs_and_spaces_expanded_false() { + let mut buf = vec![]; + format_tabs_and_spaces(0, 5, &CONFIG_E_F, &mut buf).unwrap(); + assert_eq!(buf, vec![b'\t', b' ']); + } + + #[test] + fn test_format_tabs_and_spaces_expanded_true() { + let mut buf = vec![]; + format_tabs_and_spaces(0, 5, &CONFIG_E_T, &mut buf).unwrap(); + assert_eq!(buf, vec![b' '; 5]); + } + + #[test] + fn test_format_tabs_and_spaces_from_greater_than_to() { + let mut buf = vec![]; + format_tabs_and_spaces(6, 5, &CONFIG_E_F, &mut buf).unwrap(); + assert!(buf.is_empty()); + } + + #[test] + fn test_format_from_non_zero_position() { + let mut buf = vec![]; + format_tabs_and_spaces(2, 7, &CONFIG_E_F, &mut buf).unwrap(); + assert_eq!(buf, vec![b'\t', b' ', b' ', b' ']); + } + + #[test] + fn test_multiple_full_tabs_needed() { + let mut buf = vec![]; + format_tabs_and_spaces(0, 12, &CONFIG_E_F, &mut buf).unwrap(); + assert_eq!(buf, vec![b'\t', b'\t', b'\t']); + } + + #[test] + fn test_uneven_tab_boundary_with_spaces() { + let mut buf = vec![]; + format_tabs_and_spaces(3, 10, &CONFIG_E_F, &mut buf).unwrap(); + assert_eq!(buf, vec![b'\t', b'\t', b' ', b' ']); + } + + #[test] + fn test_expanded_true_with_offset() { + let mut buf = vec![]; + format_tabs_and_spaces(3, 9, &CONFIG_E_T, &mut buf).unwrap(); + assert_eq!(buf, vec![b' '; 6]); + } + + #[test] + fn test_exact_tab_boundary_from_midpoint() { + let mut buf = vec![]; + format_tabs_and_spaces(4, 8, &CONFIG_E_F, &mut buf).unwrap(); + assert_eq!(buf, vec![b'\t']); + } + + #[test] + fn test_mixed_tabs_and_spaces_edge_case() { + let mut buf = vec![]; + format_tabs_and_spaces(5, 9, &CONFIG_E_F, &mut buf).unwrap(); + assert_eq!(buf, vec![b'\t', b' ']); + } + + #[test] + fn test_minimal_gap_with_tab() { + let mut buf = vec![]; + format_tabs_and_spaces(7, 8, &CONFIG_E_F, &mut buf).unwrap(); + assert_eq!(buf, vec![b'\t']); + } + + #[test] + fn test_expanded_false_with_tab_at_end() { + let mut buf = vec![]; + format_tabs_and_spaces(6, 8, &CONFIG_E_F, &mut buf).unwrap(); + assert_eq!(buf, vec![b'\t']); + } + } + + mod process_half_line { + use super::*; + + fn create_test_config(expanded: bool, tab_size: usize) -> Config { + Config { + sdiff_half_width: 30, + sdiff_column_two_offset: 60, + tab_size, + expanded, + separator_pos: 15, + } + } + + #[test] + fn test_empty_line_left_expanded_false() { + let config = create_test_config(false, DEF_TAB_SIZE); + let mut buf = vec![]; + process_half_line(b"", 10, false, false, &config, &mut buf).unwrap(); + assert_eq!(buf.len(), 5); + assert_eq!(buf, vec![b'\t', b'\t', b' ', b' ', b' ']); + } + + #[test] + fn test_tabs_unexpanded() { + let config = create_test_config(false, DEF_TAB_SIZE); + let mut buf = vec![]; + process_half_line(b"\tabc", 8, false, false, &config, &mut buf).unwrap(); + assert_eq!(buf, vec![b'\t', b'a', b'b', b'c', b'\t', b' ']); + } + + #[test] + fn test_utf8_multibyte() { + let config = create_test_config(false, DEF_TAB_SIZE); + let mut buf = vec![]; + let s = "😉😉😉".as_bytes(); + process_half_line(s, 3, false, false, &config, &mut buf).unwrap(); + let mut r = vec![]; + r.write_all("😉\t".as_bytes()).unwrap(); + assert_eq!(buf, r) + } + + #[test] + fn test_newline_handling() { + let config = create_test_config(false, DEF_TAB_SIZE); + let mut buf = vec![]; + process_half_line(b"abc\ndef", 5, false, false, &config, &mut buf).unwrap(); + assert_eq!(buf, vec![b'a', b'b', b'c', b'\t', b' ', b' ']); + } + + #[test] + fn test_carriage_return() { + let config = create_test_config(false, DEF_TAB_SIZE); + let mut buf = vec![]; + process_half_line(b"\rxyz", 5, true, false, &config, &mut buf).unwrap(); + let mut r = vec![b'\r']; + r.extend(vec![b'\t'; 15]); + r.extend(vec![b'x', b'y', b'z']); + assert_eq!(buf, r); + } + + #[test] + fn test_exact_width_fit() { + let config = create_test_config(true, DEF_TAB_SIZE); + let mut buf = vec![]; + process_half_line(b"abcd", 4, false, false, &config, &mut buf).unwrap(); + assert_eq!(buf.len(), 5); + assert_eq!(buf, b"abcd ".to_vec()); + } + + #[test] + fn test_non_utf8_bytes() { + let config = create_test_config(false, DEF_TAB_SIZE); + let mut buf = vec![]; + // ISO-8859-1 + process_half_line( + &[0x63, 0x61, 0x66, 0xE9], + 5, + false, + false, + &config, + &mut buf, + ) + .unwrap(); + assert_eq!(&buf, &[0x63, 0x61, 0x66, 0xE9, b' ', b' ']); + assert!(String::from_utf8(buf).is_err()); + } + + #[test] + fn test_non_utf8_bytes_ignore_padding_bytes() { + let config = create_test_config(false, DEF_TAB_SIZE); + let mut buf = vec![]; + + let utf32le_bytes = [ + 0x63, 0x00, 0x00, 0x00, // 'c' + 0x61, 0x00, 0x00, 0x00, // 'a' + 0x66, 0x00, 0x00, 0x00, // 'f' + 0xE9, 0x00, 0x00, 0x00, // 'é' + ]; + // utf8 little endiand 32 bits (or 4 bytes per char) + process_half_line(&utf32le_bytes, 6, false, false, &config, &mut buf).unwrap(); + let mut r = utf32le_bytes.to_vec(); + r.extend(vec![b' '; 3]); + assert_eq!(buf, r); + } + + #[test] + fn test_non_utf8_non_preserve_ascii_bytes_cut() { + let config = create_test_config(false, DEF_TAB_SIZE); + let mut buf = vec![]; + + let gb18030 = b"\x63\x61\x66\xA8\x80"; // some random chinese encoding + // ^ é char, start multi byte + process_half_line(gb18030, 4, false, false, &config, &mut buf).unwrap(); + assert_eq!(buf, b"\x63\x61\x66\xA8 "); // break the encoding of 'é' letter + } + + #[test] + fn test_right_line_padding() { + let config = create_test_config(false, DEF_TAB_SIZE); + let mut buf = vec![]; + process_half_line(b"xyz", 5, true, true, &config, &mut buf).unwrap(); + assert_eq!(buf.len(), 3); + } + + #[test] + fn test_mixed_tabs_spaces() { + let config = create_test_config(false, DEF_TAB_SIZE); + let mut buf = vec![]; + process_half_line(b"\t \t", 10, false, false, &config, &mut buf).unwrap(); + assert_eq!(buf, vec![b'\t', b' ', b' ', b'\t', b' ', b' ', b' ']); + } + + #[test] + fn test_overflow_multibyte() { + let config = create_test_config(false, DEF_TAB_SIZE); + let mut buf = vec![]; + let s = "日本語".as_bytes(); + process_half_line(s, 5, false, false, &config, &mut buf).unwrap(); + assert_eq!(buf, "日本 ".as_bytes()); + } + + #[test] + fn test_white_space_gutter() { + let config = create_test_config(false, DEF_TAB_SIZE); + let mut buf = vec![]; + let s = b"abc"; + process_half_line(s, 3, false, true, &config, &mut buf).unwrap(); + assert_eq!(buf, b"abc\t "); + } + + #[test] + fn test_expanded_true() { + let config = create_test_config(true, DEF_TAB_SIZE); + let mut buf = vec![]; + let s = b"abc"; + process_half_line(s, 10, false, false, &config, &mut buf).unwrap(); + assert_eq!(buf, b"abc ") + } + + #[test] + fn test_expanded_true_with_gutter() { + let config = create_test_config(true, DEF_TAB_SIZE); + let mut buf = vec![]; + let s = b"abc"; + process_half_line(s, 10, false, true, &config, &mut buf).unwrap(); + assert_eq!(buf, b"abc ") + } + + #[test] + fn test_width0_chars() { + let config = create_test_config(false, DEF_TAB_SIZE); + let mut buf = vec![]; + let s = b"abc\0\x0B\x07\x0C"; + process_half_line(s, 4, false, false, &config, &mut buf).unwrap(); + assert_eq!(buf, b"abc\0\x0B\x07\x0C\t ") + } + + #[test] + fn test_left_empty_white_space_gutter() { + let config = create_test_config(false, DEF_TAB_SIZE); + let mut buf = vec![]; + let s = b""; + process_half_line(s, 9, false, true, &config, &mut buf).unwrap(); + assert_eq!(buf, b"\t\t\t"); + } + + #[test] + fn test_s_size_eq_max_width_p1() { + let config = create_test_config(false, DEF_TAB_SIZE); + let mut buf = vec![]; + let s = b"abcdefghij"; + process_half_line(s, 10, false, false, &config, &mut buf).unwrap(); + assert_eq!(buf, b"abcdefghij "); + } + + #[test] + fn test_mixed_tabs_and_spaces_inversion() { + let config = create_test_config(false, DEF_TAB_SIZE); + let mut buf = vec![]; + let s = b" \t \t "; + process_half_line(s, 10, false, false, &config, &mut buf).unwrap(); + assert_eq!(buf, b" \t \t "); + } + + #[test] + fn test_expanded_with_tabs() { + let config = create_test_config(true, DEF_TAB_SIZE); + let mut buf = vec![]; + let s = b" \t \t "; + process_half_line(s, 10, false, false, &config, &mut buf).unwrap(); + assert_eq!(buf, b" "); + } + + #[test] + fn test_expanded_with_tabs_and_space_gutter() { + let config = create_test_config(true, DEF_TAB_SIZE); + let mut buf = vec![]; + let s = b" \t \t "; + process_half_line(s, 10, false, true, &config, &mut buf).unwrap(); + assert_eq!(buf, b" "); + } + + #[test] + fn test_zero_width_unicode_chars() { + let config = create_test_config(false, DEF_TAB_SIZE); + let mut buf = vec![]; + let s = "\u{200B}".as_bytes(); + process_half_line(s, 10, false, false, &config, &mut buf).unwrap(); + assert_eq!(buf, "\u{200B}\t\t ".as_bytes()); + } + + #[test] + fn test_multiple_carriage_returns() { + let config = create_test_config(false, DEF_TAB_SIZE); + let mut buf = vec![]; + let s = b"\r\r"; + process_half_line(s, 10, false, false, &config, &mut buf).unwrap(); + let mut r = vec![b'\r']; + r.extend(vec![b'\t'; 15]); + r.push(b'\r'); + r.extend(vec![b'\t'; 15]); + r.extend(vec![b'\t'; 2]); + r.extend(vec![b' '; 3]); + assert_eq!(buf, r); + } + + #[test] + fn test_multiple_carriage_returns_is_right_true() { + let config = create_test_config(false, DEF_TAB_SIZE); + let mut buf = vec![]; + let s = b"\r\r"; + process_half_line(s, 10, true, false, &config, &mut buf).unwrap(); + let mut r = vec![b'\r']; + r.extend(vec![b'\t'; 15]); + r.push(b'\r'); + r.extend(vec![b'\t'; 15]); + assert_eq!(buf, r); + } + + #[test] + fn test_mixed_invalid_utf8_with_valid() { + let config = create_test_config(false, DEF_TAB_SIZE); + let mut buf = vec![]; + let s = b"abc\xFF\xFEdef"; + process_half_line(s, 10, false, false, &config, &mut buf).unwrap(); + assert!(String::from_utf8(s.to_vec()).is_err()); + assert_eq!(buf, b"abc\xFF\xFEdef "); + } + + #[test] + fn test_max_width_zero() { + let config = create_test_config(false, DEF_TAB_SIZE); + let mut buf = vec![]; + let s = b"foo bar"; + process_half_line(s, 0, false, false, &config, &mut buf).unwrap(); + assert_eq!(buf, vec![b' ']); + } + + #[test] + fn test_line_only_with_tabs() { + let config = create_test_config(false, DEF_TAB_SIZE); + let mut buf = vec![]; + let s = b"\t\t\t"; + process_half_line(s, 10, false, false, &config, &mut buf).unwrap(); + assert_eq!(buf, vec![b'\t', b'\t', b' ', b' ', b' ']) + } + + #[test] + fn test_tabs_expanded() { + let config = create_test_config(true, DEF_TAB_SIZE); + let mut buf = vec![]; + let s = b"\t\t\t"; + process_half_line(s, 12, false, false, &config, &mut buf).unwrap(); + assert_eq!(buf, b" ".repeat(13)); + } + + #[test] + fn test_mixed_tabs() { + let config = create_test_config(false, DEF_TAB_SIZE); + let mut buf = vec![]; + let s = b"a\tb\tc\t"; + process_half_line(s, 10, false, false, &config, &mut buf).unwrap(); + assert_eq!(buf, b"a\tb\tc "); + } + + #[test] + fn test_mixed_tabs_with_gutter() { + let config = create_test_config(false, DEF_TAB_SIZE); + let mut buf = vec![]; + let s = b"a\tb\tc\t"; + process_half_line(s, 10, false, true, &config, &mut buf).unwrap(); + assert_eq!(buf, b"a\tb\tc\t "); + } + + #[test] + fn test_mixed_tabs_expanded() { + let config = create_test_config(true, DEF_TAB_SIZE); + let mut buf = vec![]; + let s = b"a\tb\tc\t"; + process_half_line(s, 10, false, false, &config, &mut buf).unwrap(); + assert_eq!(buf, b"a b c "); + } + + #[test] + fn test_mixed_tabs_expanded_with_gutter() { + let config = create_test_config(true, DEF_TAB_SIZE); + let mut buf = vec![]; + let s = b"a\tb\tc\t"; + process_half_line(s, 10, false, true, &config, &mut buf).unwrap(); + assert_eq!(buf, b"a b c "); + } + + #[test] + fn test_break_if_invalid_max_width() { + let config = create_test_config(true, DEF_TAB_SIZE); + let mut buf = vec![]; + let s = b"a\tb\tc\t"; + process_half_line(s, 61, false, true, &config, &mut buf).unwrap(); + assert_eq!(buf, b""); + assert_eq!(buf.len(), 0); + } + + #[test] + fn test_new_line() { + let config = create_test_config(false, DEF_TAB_SIZE); + let mut buf = vec![]; + let s = b"abc"; + process_half_line(s, 10, false, false, &config, &mut buf).unwrap(); + assert_eq!(buf, b"abc\t\t "); + } + } + + mod push_output { + // almost all behavior of the push_output was tested with tests on process_half_line + + use super::*; + + impl Default for Config { + fn default() -> Self { + Config::new(130, 8, false) + } + } + + fn create_test_config_def() -> Config { + Config::default() + } + + #[test] + fn test_left_empty_right_not_added() { + let config = create_test_config_def(); + let left_ln = b""; + let right_ln = b"bar"; + let symbol = b'>'; + let mut buf = vec![]; + push_output(&left_ln[..], &right_ln[..], symbol, &mut buf, &config).unwrap(); + assert_eq!(buf, b"\t\t\t\t\t\t\t >\tbar\n"); + } + + #[test] + fn test_right_empty_left_not_del() { + let config = create_test_config_def(); + let left_ln = b"bar"; + let right_ln = b""; + let symbol = b'>'; + let mut buf = vec![]; + push_output(&left_ln[..], &right_ln[..], symbol, &mut buf, &config).unwrap(); + assert_eq!(buf, b"bar\t\t\t\t\t\t\t >\n"); + } + + #[test] + fn test_both_empty() { + let config = create_test_config_def(); + let left_ln = b""; + let right_ln = b""; + let symbol = b' '; + let mut buf = vec![]; + push_output(&left_ln[..], &right_ln[..], symbol, &mut buf, &config).unwrap(); + assert_eq!(buf, b"\n"); + } + + #[test] + fn test_output_cut_with_maximization() { + let config = create_test_config_def(); + let left_ln = b"a".repeat(62); + let right_ln = b"a".repeat(62); + let symbol = b' '; + let mut buf = vec![]; + push_output(&left_ln[..], &right_ln[..], symbol, &mut buf, &config).unwrap(); + assert_eq!(buf.len(), 61 * 2 + 2); + assert_eq!(&buf[0..61], vec![b'a'; 61]); + assert_eq!(&buf[61..62], b"\t"); + let mut end = b"a".repeat(61); + end.push(b'\n'); + assert_eq!(&buf[62..], end); + } + + #[test] + fn test_both_lines_non_empty_with_space_symbol_max_tabs() { + let config = create_test_config_def(); + let left_ln = b"left"; + let right_ln = b"right"; + let symbol = b' '; + let mut buf = vec![]; + push_output(left_ln, right_ln, symbol, &mut buf, &config).unwrap(); + let expected_left = "left\t\t\t\t\t\t\t\t"; + let expected_right = "right"; + assert_eq!(buf, format!("{expected_left}{expected_right}\n").as_bytes()); + } + + #[test] + fn test_non_space_symbol_with_padding() { + let config = create_test_config_def(); + let left_ln = b"data"; + let right_ln = b""; + let symbol = b'<'; // impossible case, just to use different symbol + let mut buf = vec![]; + push_output(left_ln, right_ln, symbol, &mut buf, &config).unwrap(); + assert_eq!(buf, "data\t\t\t\t\t\t\t <\n".as_bytes()); + } + + #[test] + fn test_lines_exceeding_half_width() { + let config = create_test_config_def(); + let left_ln = vec![b'a'; 100]; + let left_ln = left_ln.as_slice(); + let right_ln = vec![b'b'; 100]; + let right_ln = right_ln.as_slice(); + let symbol = b' '; + let mut buf = vec![]; + push_output(left_ln, right_ln, symbol, &mut buf, &config).unwrap(); + let expected_left = "a".repeat(61); + let expected_right = "b".repeat(61); + assert_eq!(buf.len(), 61 + 1 + 61 + 1); + assert_eq!(&buf[0..61], expected_left.as_bytes()); + assert_eq!(buf[61], b'\t'); + assert_eq!(&buf[62..123], expected_right.as_bytes()); + assert_eq!(&buf[123..], b"\n"); + } + + #[test] + fn test_tabs_in_lines_expanded() { + let mut config = create_test_config_def(); + config.expanded = true; + let left_ln = b"\tleft"; + let right_ln = b"\tright"; + let symbol = b' '; + let mut buf = vec![]; + push_output(left_ln, right_ln, symbol, &mut buf, &config).unwrap(); + let expected_left = " left".to_string() + &" ".repeat(61 - 12); + let expected_right = " right"; + assert_eq!( + buf, + format!("{}{}{}\n", expected_left, " ", expected_right).as_bytes() + ); + } + + #[test] + fn test_unicode_characters() { + let config = create_test_config_def(); + let left_ln = "áéíóú".as_bytes(); + let right_ln = "😀😃😄".as_bytes(); + let symbol = b' '; + let mut buf = vec![]; + push_output(left_ln, right_ln, symbol, &mut buf, &config).unwrap(); + let expected_left = "áéíóú\t\t\t\t\t\t\t\t"; + let expected_right = "😀😃😄"; + assert_eq!(buf, format!("{expected_left}{expected_right}\n").as_bytes()); + } + } + + mod diff { + /* + Probably this hole section should be refactored when complete sdiff + arrives. I would say that these tests are more to document the + behavior of the engine than to actually test whether it is right, + because it is right, but right up to its limitations. + */ + + use super::*; + + fn generate_params() -> Params { + Params { + tabsize: 8, + expand_tabs: false, + width: 130, + ..Default::default() + } + } + + fn contains_string(vec: &[u8], s: &str) -> usize { + let pattern = s.as_bytes(); + vec.windows(pattern.len()).filter(|s| s == &pattern).count() + } + + fn calc_lines(input: &Vec) -> usize { + let mut lines_counter = 0; + + for c in input { + if c == &b'\n' { + lines_counter += 1; + } + } + + lines_counter + } + + #[test] + fn test_equal_lines() { + let params = generate_params(); + let from_file = b"equal"; + let to_file = b"equal"; + let mut output = vec![]; + diff(from_file, to_file, &mut output, ¶ms); + assert_eq!(calc_lines(&output), 1); + assert!(!output.contains(&b'<')); + assert!(!output.contains(&b'>')); + assert_eq!(contains_string(&output, "equal"), 2) + } + + // Regression for #269: the `-y` marker must sit at the gutter middle at any width. + #[test] + fn test_gutter_marker_column_wide_gutter() { + let params = Params { + tabsize: 8, + expand_tabs: true, + width: 40, + ..Default::default() + }; + let mut output = vec![]; + diff(b"aa\n", b"", &mut output, ¶ms); + let text = String::from_utf8(output).unwrap(); + let line = text.lines().next().unwrap(); + assert_eq!( + line.find('<'), + Some(19), + "gutter marker '<' should be at column 19 for --width=40, got: {line:?}" + ); + } + + #[test] + fn test_different_lines() { + let params = generate_params(); + let from_file = b"eq"; + let to_file = b"ne"; + let mut output = vec![]; + diff(from_file, to_file, &mut output, ¶ms); + assert_eq!(calc_lines(&output), 2); + assert!(output.contains(&b'>')); + assert!(output.contains(&b'<')); + assert_eq!(contains_string(&output, "eq"), 1); + assert_eq!(contains_string(&output, "ne"), 1); + } + + #[test] + fn test_added_line() { + let params = generate_params(); + let from_file = b""; + let to_file = b"new line"; + let mut output = vec![]; + diff(from_file, to_file, &mut output, ¶ms); + + assert_eq!(calc_lines(&output), 1); + assert_eq!(contains_string(&output, ">"), 1); + assert_eq!(contains_string(&output, "new line"), 1); + } + + #[test] + fn test_removed_line() { + let params = generate_params(); + let from_file = b"old line"; + let to_file = b""; + let mut output = vec![]; + diff(from_file, to_file, &mut output, ¶ms); + + assert_eq!(calc_lines(&output), 1); + assert_eq!(contains_string(&output, "<"), 1); + assert_eq!(contains_string(&output, "old line"), 1); + } + + #[test] + fn test_multiple_changes() { + let params = generate_params(); + let from_file = b"line1\nline2\nline3"; + let to_file = b"line1\nmodified\nline4"; + let mut output = vec![]; + diff(from_file, to_file, &mut output, ¶ms); + + assert_eq!(calc_lines(&output), 5); + assert_eq!(contains_string(&output, "<"), 2); + assert_eq!(contains_string(&output, ">"), 2); + } + + #[test] + fn test_unicode_and_special_chars() { + let params = generate_params(); + let from_file = "á\t€".as_bytes(); + let to_file = "€\t😊".as_bytes(); + let mut output = vec![]; + diff(from_file, to_file, &mut output, ¶ms); + + assert!(String::from_utf8_lossy(&output).contains("á")); + assert!(String::from_utf8_lossy(&output).contains("€")); + assert!(String::from_utf8_lossy(&output).contains("😊")); + assert_eq!(contains_string(&output, "<"), 1); + assert_eq!(contains_string(&output, ">"), 1); + } + + #[test] + fn test_mixed_whitespace() { + let params = generate_params(); + let from_file = b" \tspaces"; + let to_file = b"\t\t tabs"; + let mut output = vec![]; + diff(from_file, to_file, &mut output, ¶ms); + + assert!(output.contains(&b'<')); + assert!(output.contains(&b'>')); + assert!(String::from_utf8_lossy(&output).contains("spaces")); + assert!(String::from_utf8_lossy(&output).contains("tabs")); + } + + #[test] + fn test_empty_files() { + let params = generate_params(); + let from_file = b""; + let to_file = b""; + let mut output = vec![]; + diff(from_file, to_file, &mut output, ¶ms); + + assert_eq!(output, vec![]); + } + + #[test] + fn test_partially_matching_lines() { + let params = generate_params(); + let from_file = b"match\nchange"; + let to_file = b"match\nupdated"; + let mut output = vec![]; + diff(from_file, to_file, &mut output, ¶ms); + + assert_eq!(calc_lines(&output), 3); + assert_eq!(contains_string(&output, "match"), 2); + assert_eq!(contains_string(&output, "<"), 1); + assert_eq!(contains_string(&output, ">"), 1); + } + + #[test] + fn test_interleaved_add_remove() { + let params = generate_params(); + let from_file = b"A\nB\nC\nD"; + let to_file = b"B\nX\nD\nY"; + let mut output = vec![]; + diff(from_file, to_file, &mut output, ¶ms); + + assert_eq!(calc_lines(&output), 7); + assert_eq!(contains_string(&output, "A"), 1); + assert_eq!(contains_string(&output, "X"), 1); + assert_eq!(contains_string(&output, "Y"), 1); + assert_eq!(contains_string(&output, "<"), 3); + assert_eq!(contains_string(&output, ">"), 3); + } + + #[test] + fn test_swapped_lines() { + let params = generate_params(); + let from_file = b"1\n2\n3\n4"; + let to_file = b"4\n3\n2\n1"; + let mut output = vec![]; + diff(from_file, to_file, &mut output, ¶ms); + + assert_eq!(calc_lines(&output), 7); + assert_eq!(contains_string(&output, "<"), 3); + assert_eq!(contains_string(&output, ">"), 3); + } + + #[test] + fn test_gap_between_changes() { + let params = generate_params(); + let from_file = b"Start\nKeep1\nRemove\nKeep2\nEnd"; + let to_file = b"Start\nNew1\nKeep1\nKeep2\nNew2\nEnd"; + let mut output = vec![]; + diff(from_file, to_file, &mut output, ¶ms); + + assert_eq!(calc_lines(&output), 7); + assert_eq!(contains_string(&output, "Remove"), 1); + assert_eq!(contains_string(&output, "New1"), 1); + assert_eq!(contains_string(&output, "New2"), 1); + assert_eq!(contains_string(&output, "<"), 1); + assert_eq!(contains_string(&output, ">"), 2); + } + + #[test] + fn test_mixed_operations_complex() { + let params = generate_params(); + let from_file = b"Same\nOld1\nSameMid\nOld2\nSameEnd"; + let to_file = b"Same\nNew1\nSameMid\nNew2\nNew3\nSameEnd"; + let mut output = vec![]; + diff(from_file, to_file, &mut output, ¶ms); + + assert_eq!(calc_lines(&output), 8); + assert_eq!(contains_string(&output, "<"), 2); + assert_eq!(contains_string(&output, ">"), 3); + } + + #[test] + fn test_insert_remove_middle() { + let params = generate_params(); + let from_file = b"Header\nContent1\nFooter"; + let to_file = b"Header\nContent2\nFooter"; + let mut output = vec![]; + diff(from_file, to_file, &mut output, ¶ms); + + assert_eq!(calc_lines(&output), 4); + assert_eq!(contains_string(&output, "Content1"), 1); + assert_eq!(contains_string(&output, "Content2"), 1); + assert_eq!(contains_string(&output, "<"), 1); + assert_eq!(contains_string(&output, ">"), 1); + } + + #[test] + fn test_multiple_adjacent_changes() { + let params = generate_params(); + let from_file = b"A\nB\nC\nD\nE"; + let to_file = b"A\nX\nY\nD\nZ"; + let mut output = vec![]; + diff(from_file, to_file, &mut output, ¶ms); + + assert_eq!(calc_lines(&output), 8); + assert_eq!(contains_string(&output, "<"), 3); + assert_eq!(contains_string(&output, ">"), 3); + } + } + + mod config { + use super::*; + + fn create_config(full_width: usize, tab_size: usize, expanded: bool) -> Config { + Config::new(full_width, tab_size, expanded) + } + + #[test] + fn test_full_width_80_tab_4() { + let config = create_config(80, 4, false); + assert_eq!(config.sdiff_half_width, 37); + assert_eq!(config.sdiff_column_two_offset, 40); + assert_eq!(config.separator_pos, 38); + } + + #[test] + fn test_full_width_40_tab_8() { + let config = create_config(40, 8, true); + assert_eq!(config.sdiff_half_width, 16); + assert_eq!(config.sdiff_column_two_offset, 24); + assert_eq!(config.separator_pos, 19); // (16 +24 -1) /2 = 19.5 + } + + #[test] + fn test_full_width_30_tab_2() { + let config = create_config(30, 2, false); + assert_eq!(config.sdiff_half_width, 13); + assert_eq!(config.sdiff_column_two_offset, 16); + assert_eq!(config.separator_pos, 14); + } + + #[test] + fn test_small_width_10_tab_4() { + let config = create_config(10, 4, false); + assert_eq!(config.sdiff_half_width, 2); + assert_eq!(config.sdiff_column_two_offset, 8); + assert_eq!(config.separator_pos, 4); + } + + #[test] + fn test_minimal_width_3_tab_4() { + let config = create_config(3, 4, false); + assert_eq!(config.sdiff_half_width, 0); + assert_eq!(config.sdiff_column_two_offset, 3); + assert_eq!(config.separator_pos, 1); + } + + #[test] + fn test_odd_width_7_tab_3() { + let config = create_config(7, 3, false); + assert_eq!(config.sdiff_half_width, 1); + assert_eq!(config.sdiff_column_two_offset, 6); + assert_eq!(config.separator_pos, 3); + } + + #[test] + fn test_tab_size_larger_than_width() { + let config = create_config(5, 10, false); + assert_eq!(config.sdiff_half_width, 0); + assert_eq!(config.sdiff_column_two_offset, 5); + assert_eq!(config.separator_pos, 2); + } + } +} diff --git a/userland/diffutils/src/unified_diff.rs b/userland/diffutils/src/unified_diff.rs new file mode 100644 index 0000000..bd83afe --- /dev/null +++ b/userland/diffutils/src/unified_diff.rs @@ -0,0 +1,1032 @@ +// This file is part of the uutils diffutils package. +// +// For the full copyright and license information, please view the LICENSE-* +// files that was distributed with this source code. + +use std::collections::VecDeque; +use std::io::Write; + +use crate::params::Params; +use crate::utils::do_write_line; +use crate::utils::get_modification_time; + +#[derive(Debug, PartialEq)] +pub enum DiffLine { + Context(Vec), + Expected(Vec), + Actual(Vec), + MissingNL, +} + +#[derive(Debug, PartialEq)] +struct Mismatch { + pub line_number_expected: u32, + pub line_number_actual: u32, + pub lines: Vec, +} + +impl Mismatch { + fn new(line_number_expected: u32, line_number_actual: u32) -> Mismatch { + Mismatch { + line_number_expected, + line_number_actual, + lines: Vec::new(), + } + } +} + +// Produces a diff between the expected output and actual output. +fn make_diff( + expected: &[u8], + actual: &[u8], + context_size: usize, + stop_early: bool, +) -> Vec { + let mut line_number_expected = 1; + let mut line_number_actual = 1; + let mut context_queue: VecDeque<&[u8]> = VecDeque::with_capacity(context_size); + // Like GNU diff, keep changes in one hunk while at most twice the context + // size of unchanged lines separates them. + let max_gap = 2 * context_size; + let mut lines_since_mismatch = max_gap + 1; + let mut results = Vec::new(); + let mut mismatch = Mismatch::new(0, 0); + + let mut expected_lines: Vec<&[u8]> = expected.split(|&c| c == b'\n').collect(); + let mut actual_lines: Vec<&[u8]> = actual.split(|&c| c == b'\n').collect(); + + debug_assert_eq!(b"".split(|&c| c == b'\n').count(), 1); + // ^ means that underflow here is impossible + let expected_lines_count = expected_lines.len() as u32 - 1; + let actual_lines_count = actual_lines.len() as u32 - 1; + + if expected_lines.last() == Some(&&b""[..]) { + expected_lines.pop(); + } + + if actual_lines.last() == Some(&&b""[..]) { + actual_lines.pop(); + } + + for result in diff::slice(&expected_lines, &actual_lines) { + match result { + diff::Result::Left(str) => { + if lines_since_mismatch > max_gap { + results.push(mismatch); + mismatch = Mismatch::new( + line_number_expected - context_queue.len() as u32, + line_number_actual - context_queue.len() as u32, + ); + } + + while let Some(line) = context_queue.pop_front() { + mismatch.lines.push(DiffLine::Context(line.to_vec())); + } + + if mismatch.lines.last() == Some(&DiffLine::MissingNL) { + mismatch.lines.pop(); + match mismatch.lines.pop() { + Some(DiffLine::Actual(res)) => { + // We have to make sure that Actual (the + lines) + // always come after Expected (the - lines) + mismatch.lines.push(DiffLine::Expected(str.to_vec())); + if line_number_expected > expected_lines_count { + mismatch.lines.push(DiffLine::MissingNL); + } + mismatch.lines.push(DiffLine::Actual(res)); + mismatch.lines.push(DiffLine::MissingNL); + } + _ => unreachable!("unterminated Left and Common lines shouldn't be followed by more Left lines"), + } + } else { + mismatch.lines.push(DiffLine::Expected(str.to_vec())); + if line_number_expected > expected_lines_count { + mismatch.lines.push(DiffLine::MissingNL); + } + } + line_number_expected += 1; + lines_since_mismatch = 0; + } + diff::Result::Right(str) => { + if lines_since_mismatch > max_gap { + results.push(mismatch); + mismatch = Mismatch::new( + line_number_expected - context_queue.len() as u32, + line_number_actual - context_queue.len() as u32, + ); + } + + while let Some(line) = context_queue.pop_front() { + debug_assert!(mismatch.lines.last() != Some(&DiffLine::MissingNL)); + mismatch.lines.push(DiffLine::Context(line.to_vec())); + } + + mismatch.lines.push(DiffLine::Actual(str.to_vec())); + if line_number_actual > actual_lines_count { + mismatch.lines.push(DiffLine::MissingNL); + } + line_number_actual += 1; + lines_since_mismatch = 0; + } + diff::Result::Both(str, _) => { + // if one of them is missing a newline and the other isn't, then they don't actually match + if (line_number_actual > actual_lines_count) + && (line_number_expected > expected_lines_count) + { + if context_queue.len() < context_size { + while let Some(line) = context_queue.pop_front() { + debug_assert!(mismatch.lines.last() != Some(&DiffLine::MissingNL)); + mismatch.lines.push(DiffLine::Context(line.to_vec())); + } + if lines_since_mismatch < context_size { + mismatch.lines.push(DiffLine::Context(str.to_vec())); + mismatch.lines.push(DiffLine::MissingNL); + } + } + lines_since_mismatch = 0; + } else if line_number_actual > actual_lines_count { + if lines_since_mismatch > max_gap { + results.push(mismatch); + mismatch = Mismatch::new( + line_number_expected - context_queue.len() as u32, + line_number_actual - context_queue.len() as u32, + ); + } + while let Some(line) = context_queue.pop_front() { + debug_assert!(mismatch.lines.last() != Some(&DiffLine::MissingNL)); + mismatch.lines.push(DiffLine::Context(line.to_vec())); + } + mismatch.lines.push(DiffLine::Expected(str.to_vec())); + mismatch.lines.push(DiffLine::Actual(str.to_vec())); + mismatch.lines.push(DiffLine::MissingNL); + lines_since_mismatch = 0; + } else if line_number_expected > expected_lines_count { + if lines_since_mismatch > max_gap { + results.push(mismatch); + mismatch = Mismatch::new( + line_number_expected - context_queue.len() as u32, + line_number_actual - context_queue.len() as u32, + ); + } + while let Some(line) = context_queue.pop_front() { + debug_assert!(mismatch.lines.last() != Some(&DiffLine::MissingNL)); + mismatch.lines.push(DiffLine::Context(line.to_vec())); + } + mismatch.lines.push(DiffLine::Expected(str.to_vec())); + mismatch.lines.push(DiffLine::MissingNL); + mismatch.lines.push(DiffLine::Actual(str.to_vec())); + lines_since_mismatch = 0; + } else { + debug_assert!(context_queue.len() <= context_size); + if context_queue.len() >= context_size { + let _ = context_queue.pop_front(); + } + if lines_since_mismatch < context_size { + mismatch.lines.push(DiffLine::Context(str.to_vec())); + } else if context_size > 0 { + context_queue.push_back(str); + } + lines_since_mismatch += 1; + } + line_number_expected += 1; + line_number_actual += 1; + } + } + if stop_early && !results.is_empty() { + // Optimization: stop analyzing the files as soon as there are any differences + return results; + } + } + + results.push(mismatch); + results.remove(0); + + if results.is_empty() && expected_lines_count != actual_lines_count { + let mut mismatch = Mismatch::new(expected_lines.len() as u32, actual_lines.len() as u32); + // empty diff and only expected lines has a missing line at end + if expected_lines_count != expected_lines.len() as u32 { + mismatch.lines.push(DiffLine::Expected( + expected_lines + .pop() + .expect("can't be empty; produced by split()") + .to_vec(), + )); + mismatch.lines.push(DiffLine::MissingNL); + mismatch.lines.push(DiffLine::Actual( + actual_lines + .pop() + .expect("can't be empty; produced by split()") + .to_vec(), + )); + results.push(mismatch); + } else if actual_lines_count != actual_lines.len() as u32 { + mismatch.lines.push(DiffLine::Expected( + expected_lines + .pop() + .expect("can't be empty; produced by split()") + .to_vec(), + )); + mismatch.lines.push(DiffLine::Actual( + actual_lines + .pop() + .expect("can't be empty; produced by split()") + .to_vec(), + )); + mismatch.lines.push(DiffLine::MissingNL); + results.push(mismatch); + } + } + + results +} + +#[must_use] +pub fn diff(expected: &[u8], actual: &[u8], params: &Params) -> Vec { + let from_modified_time = get_modification_time(¶ms.from.to_string_lossy()); + let to_modified_time = get_modification_time(¶ms.to.to_string_lossy()); + let mut output = format!( + "--- {0}\t{1}\n+++ {2}\t{3}\n", + params.from.to_string_lossy(), + from_modified_time, + params.to.to_string_lossy(), + to_modified_time + ) + .into_bytes(); + let diff_results = make_diff(expected, actual, params.context_count, params.brief); + if diff_results.is_empty() { + return Vec::new(); + } + if params.brief { + return output; + } + for result in diff_results { + let mut line_number_expected = result.line_number_expected; + let mut line_number_actual = result.line_number_actual; + let mut expected_count = 0; + let mut actual_count = 0; + for line in &result.lines { + match line { + DiffLine::Expected(_) => { + expected_count += 1; + } + DiffLine::Context(_) => { + expected_count += 1; + actual_count += 1; + } + DiffLine::Actual(_) => { + actual_count += 1; + } + DiffLine::MissingNL => {} + } + } + // Let's imagine this diff file + // + // --- a/something + // +++ b/something + // @@ -2,0 +3,1 @@ + // + x + // + // In the unified diff format as implemented by GNU diff and patch, + // this is an instruction to insert the x *after* the preexisting line 2, + // not before. You can demonstrate it this way: + // + // $ echo -ne '--- a/something\t\n+++ b/something\t\n@@ -2,0 +3,1 @@\n+ x\n' > diff + // $ echo -ne 'a\nb\nc\nd\n' > something + // $ patch -p1 < diff + // patching file something + // $ cat something + // a + // b + // x + // c + // d + // + // Notice how the x winds up at line 3, not line 2. This requires contortions to + // work with our diffing algorithm, which keeps track of the "intended destination line", + // not a line that things are supposed to be placed after. It's changing the first number, + // not the second, that actually affects where the x goes. + // + // # change the first number from 2 to 3, and now the x is on line 4 (it's placed after line 3) + // $ echo -ne '--- a/something\t\n+++ b/something\t\n@@ -3,0 +3,1 @@\n+ x\n' > diff + // $ echo -ne 'a\nb\nc\nd\n' > something + // $ patch -p1 < diff + // patching file something + // $ cat something + // a + // b + // c + // x + // d + // # change the third number from 3 to 1000, and it's obvious that it's the first number that's + // # actually being read + // $ echo -ne '--- a/something\t\n+++ b/something\t\n@@ -2,0 +1000,1 @@\n+ x\n' > diff + // $ echo -ne 'a\nb\nc\nd\n' > something + // $ patch -p1 < diff + // patching file something + // $ cat something + // a + // b + // x + // c + // d + // + // Now watch what happens if I add a context line: + // + // $ echo -ne '--- a/something\t\n+++ b/something\t\n@@ -2,1 +3,2 @@\n+ x\n c\n' > diff + // $ echo -ne 'a\nb\nc\nd\n' > something + // $ patch -p1 < diff + // patching file something + // Hunk #1 succeeded at 3 (offset 1 line). + // + // It technically "succeeded", but this is a warning. We want to produce clean diffs. + // Now that I have a context line, I'm supposed to say what line it's actually on, which is the + // line that the x will wind up on, and not the line immediately before. + // + // $ echo -ne '--- a/something\t\n+++ b/something\t\n@@ -3,1 +3,2 @@\n+ x\n c\n' > diff + // $ echo -ne 'a\nb\nc\nd\n' > something + // $ patch -p1 < diff + // patching file something + // $ cat something + // a + // b + // x + // c + // d + // + // I made this comment because this stuff is not obvious from GNU's + // documentation on the format at all. + if expected_count == 0 { + line_number_expected -= 1; + } + if actual_count == 0 { + line_number_actual -= 1; + } + let exp_ct = if expected_count == 1 { + String::new() + } else { + format!(",{expected_count}") + }; + let act_ct = if actual_count == 1 { + String::new() + } else { + format!(",{actual_count}") + }; + writeln!( + output, + "@@ -{line_number_expected}{exp_ct} +{line_number_actual}{act_ct} @@" + ) + .expect("write to Vec is infallible"); + for line in result.lines { + match line { + DiffLine::Expected(e) => { + write!(output, "-").expect("write to Vec is infallible"); + do_write_line(&mut output, &e, params.expand_tabs, params.tabsize) + .expect("write to Vec is infallible"); + writeln!(output).unwrap(); + } + DiffLine::Context(c) => { + write!(output, " ").expect("write to Vec is infallible"); + do_write_line(&mut output, &c, params.expand_tabs, params.tabsize) + .expect("write to Vec is infallible"); + writeln!(output).unwrap(); + } + DiffLine::Actual(r) => { + write!(output, "+",).expect("write to Vec is infallible"); + do_write_line(&mut output, &r, params.expand_tabs, params.tabsize) + .expect("write to Vec is infallible"); + writeln!(output).unwrap(); + } + DiffLine::MissingNL => { + writeln!(output, r"\ No newline at end of file") + .expect("write to Vec is infallible"); + } + } + } + } + output +} + +#[cfg(test)] +mod tests { + use super::*; + use pretty_assertions::assert_eq; + + use crate::utils::testcmds::PATCH_CMD; + + #[test] + fn test_permutations() { + let target = "target/unified-diff/"; + // test all possible six-line files. + let _ = std::fs::create_dir(target); + for &a in &[0, 1, 2] { + for &b in &[0, 1, 2] { + for &c in &[0, 1, 2] { + for &d in &[0, 1, 2] { + for &e in &[0, 1, 2] { + for &f in &[0, 1, 2] { + use std::fs::{self, File}; + use std::io::Write; + let mut alef = Vec::new(); + let mut bet = Vec::new(); + alef.write_all(if a == 0 { b"a\n" } else { b"b\n" }) + .unwrap(); + if a != 2 { + bet.write_all(b"b\n").unwrap(); + } + alef.write_all(if b == 0 { b"c\n" } else { b"d\n" }) + .unwrap(); + if b != 2 { + bet.write_all(b"d\n").unwrap(); + } + alef.write_all(if c == 0 { b"e\n" } else { b"f\n" }) + .unwrap(); + if c != 2 { + bet.write_all(b"f\n").unwrap(); + } + alef.write_all(if d == 0 { b"g\n" } else { b"h\n" }) + .unwrap(); + if d != 2 { + bet.write_all(b"h\n").unwrap(); + } + alef.write_all(if e == 0 { b"i\n" } else { b"j\n" }) + .unwrap(); + if e != 2 { + bet.write_all(b"j\n").unwrap(); + } + alef.write_all(if f == 0 { b"k\n" } else { b"l\n" }) + .unwrap(); + if f != 2 { + bet.write_all(b"l\n").unwrap(); + } + // This test diff is intentionally reversed. + // We want it to turn the alef into bet. + let patched = &format!("{target}/alef"); + let diff = diff( + &alef, + &bet, + &Params { + from: patched.into(), + to: patched.into(), + context_count: 2, + ..Default::default() + }, + ); + File::create(format!("{target}/ab.diff")) + .unwrap() + .write_all(&diff) + .unwrap(); + let mut fa = File::create(format!("{target}/alef")).unwrap(); + fa.write_all(&alef[..]).unwrap(); + let mut fb = File::create(format!("{target}/bet")).unwrap(); + fb.write_all(&bet[..]).unwrap(); + let _ = fa; + let _ = fb; + println!( + "diff: {:?}", + String::from_utf8(diff.clone()) + .unwrap_or_else(|_| String::from("[Invalid UTF-8]")) + ); + println!( + "alef: {:?}", + String::from_utf8(alef.clone()) + .unwrap_or_else(|_| String::from("[Invalid UTF-8]")) + ); + println!( + "bet: {:?}", + String::from_utf8(bet.clone()) + .unwrap_or_else(|_| String::from("[Invalid UTF-8]")) + ); + + use crate::utils::testcmds::PATCH_CMD; + + let output = PATCH_CMD + .new() + .arg("-p0") + .stdin(File::open(format!("{target}/ab.diff")).unwrap()) + .output() + .unwrap(); + println!("{}", String::from_utf8_lossy(&output.stdout)); + println!("{}", String::from_utf8_lossy(&output.stderr)); + assert!(output.status.success(), "{output:?}"); + let alef = fs::read(format!("{target}/alef")).unwrap(); + assert_eq!(alef, bet); + } + } + } + } + } + } + } + + #[test] + fn test_permutations_missing_line_ending() { + let target = "target/unified-diff/"; + // test all possible six-line files with missing newlines. + let _ = std::fs::create_dir(target); + for &a in &[0, 1, 2] { + for &b in &[0, 1, 2] { + for &c in &[0, 1, 2] { + for &d in &[0, 1, 2] { + for &e in &[0, 1, 2] { + for &f in &[0, 1, 2] { + for &g in &[0, 1, 2] { + use std::fs::{self, File}; + use std::io::Write; + let mut alef = Vec::new(); + let mut bet = Vec::new(); + alef.write_all(if a == 0 { b"a\n" } else { b"b\n" }) + .unwrap(); + if a != 2 { + bet.write_all(b"b\n").unwrap(); + } + alef.write_all(if b == 0 { b"c\n" } else { b"d\n" }) + .unwrap(); + if b != 2 { + bet.write_all(b"d\n").unwrap(); + } + alef.write_all(if c == 0 { b"e\n" } else { b"f\n" }) + .unwrap(); + if c != 2 { + bet.write_all(b"f\n").unwrap(); + } + alef.write_all(if d == 0 { b"g\n" } else { b"h\n" }) + .unwrap(); + if d != 2 { + bet.write_all(b"h\n").unwrap(); + } + alef.write_all(if e == 0 { b"i\n" } else { b"j\n" }) + .unwrap(); + if e != 2 { + bet.write_all(b"j\n").unwrap(); + } + alef.write_all(if f == 0 { b"k\n" } else { b"l\n" }) + .unwrap(); + if f != 2 { + bet.write_all(b"l\n").unwrap(); + } + match g { + 0 => { + alef.pop(); + } + 1 => { + bet.pop(); + } + 2 => { + alef.pop(); + bet.pop(); + } + _ => unreachable!(), + } + // This test diff is intentionally reversed. + // We want it to turn the alef into bet. + let patched = &format!("{target}/alefn"); + let diff = diff( + &alef, + &bet, + &Params { + from: patched.into(), + to: patched.into(), + context_count: 2, + ..Default::default() + }, + ); + File::create(format!("{target}/abn.diff")) + .unwrap() + .write_all(&diff) + .unwrap(); + let mut fa = File::create(format!("{target}/alefn")).unwrap(); + fa.write_all(&alef[..]).unwrap(); + let mut fb = File::create(format!("{target}/betn")).unwrap(); + fb.write_all(&bet[..]).unwrap(); + let _ = fa; + let _ = fb; + let output = PATCH_CMD + .new() + .arg("-p0") + .stdin(File::open(format!("{target}/abn.diff")).unwrap()) + .output() + .unwrap(); + assert!(output.status.success(), "{output:?}"); + //println!("{}", String::from_utf8_lossy(&output.stdout)); + //println!("{}", String::from_utf8_lossy(&output.stderr)); + let alef = fs::read(format!("{target}/alefn")).unwrap(); + assert_eq!(alef, bet); + } + } + } + } + } + } + } + } + + #[test] + fn test_permutations_empty_lines() { + let target = "target/unified-diff/"; + // test all possible six-line files with missing newlines. + let _ = std::fs::create_dir(target); + for &a in &[0, 1, 2] { + for &b in &[0, 1, 2] { + for &c in &[0, 1, 2] { + for &d in &[0, 1, 2] { + for &e in &[0, 1, 2] { + for &f in &[0, 1, 2] { + for &g in &[0, 1, 2, 3] { + use std::fs::{self, File}; + use std::io::Write; + let mut alef = Vec::new(); + let mut bet = Vec::new(); + alef.write_all(if a == 0 { b"\n" } else { b"b\n" }).unwrap(); + if a != 2 { + bet.write_all(b"b\n").unwrap(); + } + alef.write_all(if b == 0 { b"\n" } else { b"d\n" }).unwrap(); + if b != 2 { + bet.write_all(b"d\n").unwrap(); + } + alef.write_all(if c == 0 { b"\n" } else { b"f\n" }).unwrap(); + if c != 2 { + bet.write_all(b"f\n").unwrap(); + } + alef.write_all(if d == 0 { b"\n" } else { b"h\n" }).unwrap(); + if d != 2 { + bet.write_all(b"h\n").unwrap(); + } + alef.write_all(if e == 0 { b"\n" } else { b"j\n" }).unwrap(); + if e != 2 { + bet.write_all(b"j\n").unwrap(); + } + alef.write_all(if f == 0 { b"\n" } else { b"l\n" }).unwrap(); + if f != 2 { + bet.write_all(b"l\n").unwrap(); + } + match g { + 0 => { + alef.pop(); + } + 1 => { + bet.pop(); + } + 2 => { + alef.pop(); + bet.pop(); + } + 3 => {} + _ => unreachable!(), + } + // This test diff is intentionally reversed. + // We want it to turn the alef into bet. + let patched = &format!("{target}/alef_"); + let diff = diff( + &alef, + &bet, + &Params { + from: patched.into(), + to: patched.into(), + context_count: 2, + ..Default::default() + }, + ); + File::create(format!("{target}/ab_.diff")) + .unwrap() + .write_all(&diff) + .unwrap(); + let mut fa = File::create(format!("{target}/alef_")).unwrap(); + fa.write_all(&alef[..]).unwrap(); + let mut fb = File::create(format!("{target}/bet_")).unwrap(); + fb.write_all(&bet[..]).unwrap(); + let _ = fa; + let _ = fb; + let output = PATCH_CMD + .new() + .arg("-p0") + .stdin(File::open(format!("{target}/ab_.diff")).unwrap()) + .output() + .unwrap(); + assert!(output.status.success(), "{output:?}"); + //println!("{}", String::from_utf8_lossy(&output.stdout)); + //println!("{}", String::from_utf8_lossy(&output.stderr)); + let alef = fs::read(format!("{target}/alef_")).unwrap(); + assert_eq!(alef, bet); + } + } + } + } + } + } + } + } + + #[test] + fn test_permutations_missing_lines() { + let target = "target/unified-diff/"; + // test all possible six-line files. + let _ = std::fs::create_dir(target); + for &a in &[0, 1, 2] { + for &b in &[0, 1, 2] { + for &c in &[0, 1, 2] { + for &d in &[0, 1, 2] { + for &e in &[0, 1, 2] { + for &f in &[0, 1, 2] { + use std::fs::{self, File}; + use std::io::Write; + let mut alef = Vec::new(); + let mut bet = Vec::new(); + alef.write_all(if a == 0 { b"a\n" } else { b"" }).unwrap(); + if a != 2 { + bet.write_all(b"b\n").unwrap(); + } + alef.write_all(if b == 0 { b"c\n" } else { b"" }).unwrap(); + if b != 2 { + bet.write_all(b"d\n").unwrap(); + } + alef.write_all(if c == 0 { b"e\n" } else { b"" }).unwrap(); + if c != 2 { + bet.write_all(b"f\n").unwrap(); + } + alef.write_all(if d == 0 { b"g\n" } else { b"" }).unwrap(); + if d != 2 { + bet.write_all(b"h\n").unwrap(); + } + alef.write_all(if e == 0 { b"i\n" } else { b"" }).unwrap(); + if e != 2 { + bet.write_all(b"j\n").unwrap(); + } + alef.write_all(if f == 0 { b"k\n" } else { b"" }).unwrap(); + if f != 2 { + bet.write_all(b"l\n").unwrap(); + } + // This test diff is intentionally reversed. + // We want it to turn the alef into bet. + let patched = &format!("{target}/alefx"); + let diff = diff( + &alef, + &bet, + &Params { + from: patched.into(), + to: patched.into(), + context_count: 2, + ..Default::default() + }, + ); + File::create(format!("{target}/abx.diff")) + .unwrap() + .write_all(&diff) + .unwrap(); + let mut fa = File::create(format!("{target}/alefx")).unwrap(); + fa.write_all(&alef[..]).unwrap(); + let mut fb = File::create(format!("{target}/betx")).unwrap(); + fb.write_all(&bet[..]).unwrap(); + let _ = fa; + let _ = fb; + let output = PATCH_CMD + .new() + .arg("-p0") + .stdin(File::open(format!("{target}/abx.diff")).unwrap()) + .output() + .unwrap(); + assert!(output.status.success(), "{output:?}"); + //println!("{}", String::from_utf8_lossy(&output.stdout)); + //println!("{}", String::from_utf8_lossy(&output.stderr)); + let alef = fs::read(format!("{target}/alefx")).unwrap(); + assert_eq!(alef, bet); + } + } + } + } + } + } + } + + #[test] + fn test_permutations_reverse() { + let target = "target/unified-diff/"; + // test all possible six-line files. + let _ = std::fs::create_dir(target); + for &a in &[0, 1, 2] { + for &b in &[0, 1, 2] { + for &c in &[0, 1, 2] { + for &d in &[0, 1, 2] { + for &e in &[0, 1, 2] { + for &f in &[0, 1, 2] { + use std::fs::{self, File}; + use std::io::Write; + let mut alef = Vec::new(); + let mut bet = Vec::new(); + alef.write_all(if a == 0 { b"a\n" } else { b"f\n" }) + .unwrap(); + if a != 2 { + bet.write_all(b"a\n").unwrap(); + } + alef.write_all(if b == 0 { b"b\n" } else { b"e\n" }) + .unwrap(); + if b != 2 { + bet.write_all(b"b\n").unwrap(); + } + alef.write_all(if c == 0 { b"c\n" } else { b"d\n" }) + .unwrap(); + if c != 2 { + bet.write_all(b"c\n").unwrap(); + } + alef.write_all(if d == 0 { b"d\n" } else { b"c\n" }) + .unwrap(); + if d != 2 { + bet.write_all(b"d\n").unwrap(); + } + alef.write_all(if e == 0 { b"e\n" } else { b"b\n" }) + .unwrap(); + if e != 2 { + bet.write_all(b"e\n").unwrap(); + } + alef.write_all(if f == 0 { b"f\n" } else { b"a\n" }) + .unwrap(); + if f != 2 { + bet.write_all(b"f\n").unwrap(); + } + // This test diff is intentionally reversed. + // We want it to turn the alef into bet. + let patched = &format!("{target}/alefr"); + let diff = diff( + &alef, + &bet, + &Params { + from: patched.into(), + to: patched.into(), + context_count: 2, + ..Default::default() + }, + ); + File::create(format!("{target}/abr.diff")) + .unwrap() + .write_all(&diff) + .unwrap(); + let mut fa = File::create(format!("{target}/alefr")).unwrap(); + fa.write_all(&alef[..]).unwrap(); + let mut fb = File::create(format!("{target}/betr")).unwrap(); + fb.write_all(&bet[..]).unwrap(); + let _ = fa; + let _ = fb; + let output = PATCH_CMD + .new() + .arg("-p0") + .stdin(File::open(format!("{target}/abr.diff")).unwrap()) + .output() + .unwrap(); + assert!(output.status.success(), "{output:?}"); + //println!("{}", String::from_utf8_lossy(&output.stdout)); + //println!("{}", String::from_utf8_lossy(&output.stderr)); + let alef = fs::read(format!("{target}/alefr")).unwrap(); + assert_eq!(alef, bet); + } + } + } + } + } + } + } + + #[test] + fn test_stop_early() { + use crate::assert_diff_eq; + + let from_filename = "foo"; + let from = ["a", "b", "c", ""].join("\n"); + let to_filename = "bar"; + let to = ["a", "d", "c", ""].join("\n"); + + let diff_full = diff( + from.as_bytes(), + to.as_bytes(), + &Params { + from: from_filename.into(), + to: to_filename.into(), + ..Default::default() + }, + ); + + let expected_full = [ + "--- foo\tTIMESTAMP", + "+++ bar\tTIMESTAMP", + "@@ -1,3 +1,3 @@", + " a", + "-b", + "+d", + " c", + "", + ] + .join("\n"); + assert_diff_eq!(diff_full, expected_full); + + let diff_brief = diff( + from.as_bytes(), + to.as_bytes(), + &Params { + from: from_filename.into(), + to: to_filename.into(), + brief: true, + ..Default::default() + }, + ); + + let expected_brief = ["--- foo\tTIMESTAMP", "+++ bar\tTIMESTAMP", ""].join("\n"); + assert_diff_eq!(diff_brief, expected_brief); + + let nodiff_full = diff( + from.as_bytes(), + from.as_bytes(), + &Params { + from: from_filename.into(), + to: to_filename.into(), + ..Default::default() + }, + ); + assert!(nodiff_full.is_empty()); + + let nodiff_brief = diff( + from.as_bytes(), + from.as_bytes(), + &Params { + from: from_filename.into(), + to: to_filename.into(), + brief: true, + ..Default::default() + }, + ); + assert!(nodiff_brief.is_empty()); + } + + #[test] + fn test_hunk_merging() { + use crate::assert_diff_eq; + + let params = Params { + from: "foo".into(), + to: "bar".into(), + context_count: 1, + ..Default::default() + }; + + // Changes separated by up to twice the context size share one hunk. + let from = ["a", "b", "c", "d", "e", "f", ""].join("\n"); + let to = ["a", "B", "c", "d", "E", "f", ""].join("\n"); + let expected = [ + "--- foo\tTIMESTAMP", + "+++ bar\tTIMESTAMP", + "@@ -1,6 +1,6 @@", + " a", + "-b", + "+B", + " c", + " d", + "-e", + "+E", + " f", + "", + ] + .join("\n"); + let output = diff(from.as_bytes(), to.as_bytes(), ¶ms); + assert_diff_eq!(output, expected); + + // One more unchanged line between them splits the hunk. + let from = ["a", "b", "c", "d", "e", "f", "g", ""].join("\n"); + let to = ["a", "B", "c", "d", "e", "F", "g", ""].join("\n"); + let expected = [ + "--- foo\tTIMESTAMP", + "+++ bar\tTIMESTAMP", + "@@ -1,3 +1,3 @@", + " a", + "-b", + "+B", + " c", + "@@ -5,3 +5,3 @@", + " e", + "-f", + "+F", + " g", + "", + ] + .join("\n"); + let output = diff(from.as_bytes(), to.as_bytes(), ¶ms); + assert_diff_eq!(output, expected); + + // An insertion before a nearby change gives the two sides different + // line counts; the hunks must still be merged (issue #268). + let from = ["milk", "eggs", "bread", "butter", "apples", ""].join("\n"); + let to = ["milk", "eggs", "yogurt", "bread", "honey", "apples", ""].join("\n"); + let expected = [ + "--- foo\tTIMESTAMP", + "+++ bar\tTIMESTAMP", + "@@ -2,4 +2,5 @@", + " eggs", + "+yogurt", + " bread", + "-butter", + "+honey", + " apples", + "", + ] + .join("\n"); + let output = diff(from.as_bytes(), to.as_bytes(), ¶ms); + assert_diff_eq!(output, expected); + } +} diff --git a/userland/diffutils/src/utils.rs b/userland/diffutils/src/utils.rs new file mode 100644 index 0000000..d9415af --- /dev/null +++ b/userland/diffutils/src/utils.rs @@ -0,0 +1,300 @@ +// This file is part of the uutils diffutils package. +// +// For the full copyright and license information, please view the LICENSE-* +// files that was distributed with this source code. + +use regex::Regex; +use std::{ffi::OsString, io::Write}; +use unicode_width::UnicodeWidthStr; + +/// Replace tabs by spaces in the input line. +/// Correctly handle multi-bytes characters. +/// This assumes that line does not contain any line breaks (if it does, the result is undefined). +#[must_use] +pub fn do_expand_tabs(line: &[u8], tabsize: usize) -> Vec { + let tab = b'\t'; + let ntabs = line.iter().filter(|c| **c == tab).count(); + if ntabs == 0 { + return line.to_vec(); + } + let mut result = Vec::with_capacity(line.len() + ntabs * (tabsize - 1)); + let mut offset = 0; + + let mut iter = line.split(|c| *c == tab).peekable(); + while let Some(chunk) = iter.next() { + match String::from_utf8(chunk.to_vec()) { + Ok(s) => offset += UnicodeWidthStr::width(s.as_str()), + Err(_) => offset += chunk.len(), + } + result.extend_from_slice(chunk); + if iter.peek().is_some() { + result.resize(result.len() + tabsize - offset % tabsize, b' '); + offset = 0; + } + } + + result +} + +/// Write a single line to an output stream, expanding tabs to space if necessary. +/// This assumes that line does not contain any line breaks +/// (if it does and tabs are to be expanded to spaces, the result is undefined). +pub fn do_write_line( + output: &mut Vec, + line: &[u8], + expand_tabs: bool, + tabsize: usize, +) -> std::io::Result<()> { + if expand_tabs { + output.write_all(do_expand_tabs(line, tabsize).as_slice()) + } else { + output.write_all(line) + } +} + +/// Retrieves the modification time of the input file specified by file path +/// If an error occurs, it returns the current system time +pub fn get_modification_time(file_path: &str) -> String { + use chrono::{DateTime, Local}; + use std::fs; + use std::time::SystemTime; + + let modification_time: SystemTime = fs::metadata(file_path) + .and_then(|m| m.modified()) + .unwrap_or(SystemTime::now()); + + let modification_time: DateTime = modification_time.into(); + let modification_time: String = modification_time + .format("%Y-%m-%d %H:%M:%S%.9f %z") + .to_string(); + + modification_time +} + +pub fn format_failure_to_read_input_file( + executable: &OsString, + filepath: &OsString, + error: &std::io::Error, +) -> String { + // std::io::Error's display trait outputs "{detail} (os error {code})" + // but we want only the {detail} (error string) part + let error_code_re = Regex::new(r"\ \(os\ error\ \d+\)$").unwrap(); + format!( + "{}: {}: {}", + executable.to_string_lossy(), + filepath.to_string_lossy(), + error_code_re.replace(error.to_string().as_str(), ""), + ) +} + +pub fn report_failure_to_read_input_file( + executable: &OsString, + filepath: &OsString, + error: &std::io::Error, +) { + eprintln!( + "{}", + format_failure_to_read_input_file(executable, filepath, error) + ); +} + +#[cfg(test)] +pub mod testcmds { + // Command construction wrapper that provides some validation and non-obscure, "fail fast" + // feedback and error messages. + use std::any::Any; + use std::io::Write; + use std::panic::catch_unwind; + use std::process::{Command, Stdio}; + use std::sync::LazyLock; + + pub struct CmdFactory { + cmd: &'static str, + validated_once: LazyLock>, + validate: fn(&CmdFactory) -> (), + } + + impl CmdFactory { + pub fn new(&self) -> Command { + match &*self.validated_once { + Ok(()) => Command::new(self.cmd), + Err(errmsg) => panic!( + "'{}' validation failed in earlier thread/test: {}", + self.cmd, errmsg + ), + } + } + // "self" is not compatible with static initialization + fn try_catch_validate(cmd: &CmdFactory) -> Result<(), String> { + // Note catch_unwind() does _not_ hide error messages, stack traces, etc. + catch_unwind(|| { + let _ = (cmd.validate)(cmd); + }) + .map_err(find_panic_message) + } + } + + fn find_panic_message(payload: Box) -> String { + // https://github.com/rust-lang/rust/blob/1.95.0/library/std/src/panicking.rs#L771 + if let Some(&s) = payload.downcast_ref::<&'static str>() { + String::from(s) + } else if let Some(s) = payload.downcast_ref::() { + s.clone() + } else { + format!( + "Unusual panic payload type {:?}, look for the first thread/test that failed", + payload.type_id(), + ) + } + } + + pub static PATCH_CMD: CmdFactory = CmdFactory { + cmd: if cfg!(target_os = "macos") { + "gpatch" // brew install gpatch + } else { + "patch" + }, + validated_once: LazyLock::new(|| CmdFactory::try_catch_validate(&PATCH_CMD)), + + validate: (|myself| { + let output = Command::new(myself.cmd) + .arg("--version") + .output() + .expect(format!("`{} --version` failed", myself.cmd).as_str()); + // Non-GNU versions have subtle differences. When some newlines are missing in some test + // patches, the macOS version can even stall the whole test run. + assert!(output.stdout.starts_with(b"GNU patch")); + assert!(output.status.success()); + }), + }; + + pub static ED_CMD: CmdFactory = CmdFactory { + cmd: "ed", + validated_once: LazyLock::new(|| CmdFactory::try_catch_validate(&ED_CMD)), + + validate: (|myself| { + let mut child = Command::new(myself.cmd) + .arg("!echo hello_ed") + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .spawn() + .expect("Failed to start 'ed' command"); + + let mut stdin = child.stdin.take().unwrap(); + writeln!(stdin, "1p\nq").expect("Failed to send command to 'ed'"); + + let output = child + .wait_with_output() + .expect("Failed to read 'ed' stdout"); + assert_eq!(String::from_utf8_lossy(&output.stdout), "9\nhello_ed\n"); + }), + }; +} + +#[cfg(test)] +mod tests { + use super::*; + + mod expand_tabs { + use super::*; + use pretty_assertions::assert_eq; + + fn assert_tab_expansion(line: &str, tabsize: usize, expected: &str) { + assert_eq!( + do_expand_tabs(line.as_bytes(), tabsize), + expected.as_bytes() + ); + } + + #[test] + fn basics() { + assert_tab_expansion("foo barr baz", 8, "foo barr baz"); + assert_tab_expansion("foo\tbarr\tbaz", 8, "foo barr baz"); + assert_tab_expansion("foo\tbarr\tbaz", 5, "foo barr baz"); + assert_tab_expansion("foo\tbarr\tbaz", 2, "foo barr baz"); + } + + #[test] + fn multibyte_chars() { + assert_tab_expansion("foo\tépée\tbaz", 8, "foo épée baz"); + assert_tab_expansion("foo\t😉\tbaz", 5, "foo 😉 baz"); + + // Note: The Woman Scientist emoji (👩â€ðŸ”¬) is a ZWJ sequence combining + // the Woman emoji (👩) and the Microscope emoji (🔬). On supported platforms + // it is displayed as a single emoji and has a print size of 2 columns. + // Terminal emulators tend to not support this, and display the two emojis + // side by side, thus accounting for a print size of 4 columns, but the + // unicode_width crate reports a correct size of 2. + assert_tab_expansion("foo\t👩â€ðŸ”¬\tbaz", 6, "foo 👩â€ðŸ”¬ baz"); + } + + #[test] + fn invalid_utf8() { + // [240, 240, 152, 137] is an invalid UTF-8 sequence, so it is handled as 4 bytes + assert_eq!( + do_expand_tabs(&[240, 240, 152, 137, 9, 102, 111, 111], 8), + &[240, 240, 152, 137, 32, 32, 32, 32, 102, 111, 111] + ); + } + } + + mod write_line { + use super::*; + use pretty_assertions::assert_eq; + + fn assert_line_written(line: &str, expand_tabs: bool, tabsize: usize, expected: &str) { + let mut output: Vec = Vec::new(); + assert!(do_write_line(&mut output, line.as_bytes(), expand_tabs, tabsize).is_ok()); + assert_eq!(output, expected.as_bytes()); + } + + #[test] + fn basics() { + assert_line_written("foo bar baz", false, 8, "foo bar baz"); + assert_line_written("foo bar\tbaz", false, 8, "foo bar\tbaz"); + assert_line_written("foo bar\tbaz", true, 8, "foo bar baz"); + } + } + + mod modification_time { + use super::*; + + #[test] + fn set_time() { + use chrono::{DateTime, Local}; + use std::time::SystemTime; + use tempfile::NamedTempFile; + + let temp = NamedTempFile::new().unwrap(); + // set file modification time equal to current time + let current = SystemTime::now(); + let _ = temp.as_file().set_modified(current); + + // format current time + let current: DateTime = current.into(); + let current: String = current.format("%Y-%m-%d %H:%M:%S%.9f %z").to_string(); + + // verify + assert_eq!( + current, + get_modification_time(&temp.path().to_string_lossy()) + ); + } + + #[test] + fn invalid_file() { + use chrono::{DateTime, Local}; + use std::time::SystemTime; + + let invalid_file = "target/utils/invalid-file"; + + // store current time before calling `get_modification_time` + // Because the file is invalid, it will return SystemTime::now() + // which will be greater than previously saved time + let current_time: DateTime = SystemTime::now().into(); + let m_time: DateTime = get_modification_time(invalid_file).parse().unwrap(); + + assert!(m_time > current_time); + } + } +} diff --git a/userland/diffutils/tests/integration.rs b/userland/diffutils/tests/integration.rs new file mode 100644 index 0000000..ad5efd9 --- /dev/null +++ b/userland/diffutils/tests/integration.rs @@ -0,0 +1,945 @@ +// This file is part of the uutils diffutils package. +// +// For the full copyright and license information, please view the LICENSE-* +// files that was distributed with this source code. + +use assert_cmd::cargo::cargo_bin_cmd; +use predicates::prelude::*; +use std::fs::File; +#[cfg(not(windows))] +use std::fs::OpenOptions; +use std::io::Write; +use tempfile::{tempdir, NamedTempFile}; + +// Integration tests for the diffutils command +mod common { + use super::*; + + #[test] + fn unknown_param() -> Result<(), Box> { + let mut cmd = cargo_bin_cmd!("diffutils"); + cmd.arg("patch"); + cmd.assert() + .code(predicate::eq(2)) + .failure() + .stderr(predicate::eq("patch: utility not supported\n")); + + let mut cmd = cargo_bin_cmd!("diffutils"); + cmd.assert() + .code(predicate::eq(0)) + .success() + .stderr(predicate::str::starts_with( + "Expected utility name as second argument, got nothing.\n", + )); + + for subcmd in ["diff", "cmp", "uu-diff", "uucmp"] { + let mut cmd = cargo_bin_cmd!("diffutils"); + cmd.arg(subcmd); + cmd.arg("--foobar"); + cmd.assert() + .code(predicate::eq(2)) + .failure() + .stderr(predicate::str::contains("unrecognized option '--foobar'")); + } + Ok(()) + } + + #[test] + fn cannot_read_files() -> Result<(), Box> { + let file = NamedTempFile::new()?; + + let nofile = NamedTempFile::new()?; + let nopath = nofile.into_temp_path(); + std::fs::remove_file(&nopath)?; + + #[cfg(not(windows))] + let error_message = "No such file or directory"; + #[cfg(windows)] + let error_message = "The system cannot find the file specified."; + + for subcmd in ["diff", "cmp"] { + let mut cmd = cargo_bin_cmd!("diffutils"); + cmd.arg(subcmd); + cmd.arg(&nopath).arg(file.path()); + cmd.assert() + .code(predicate::eq(2)) + .failure() + .stderr(predicate::str::ends_with(format!( + ": {}: {error_message}\n", + &nopath.as_os_str().to_string_lossy() + ))); + + let mut cmd = cargo_bin_cmd!("diffutils"); + cmd.arg(subcmd); + cmd.arg(file.path()).arg(&nopath); + cmd.assert() + .code(predicate::eq(2)) + .failure() + .stderr(predicate::str::ends_with(format!( + ": {}: {error_message}\n", + &nopath.as_os_str().to_string_lossy() + ))); + } + + let mut cmd = cargo_bin_cmd!("diffutils"); + cmd.arg("diff"); + cmd.arg(&nopath).arg(&nopath); + cmd.assert().code(predicate::eq(2)).failure().stderr( + predicate::str::contains(format!( + ": {}: {error_message}\n", + &nopath.as_os_str().to_string_lossy() + )) + .count(2), + ); + + Ok(()) + } +} + +mod diff { + use diffutilslib::assert_diff_eq; + + use super::*; + + #[test] + fn no_differences() -> Result<(), Box> { + let file = NamedTempFile::new()?; + for option in ["", "-u", "-c", "-e"] { + let mut cmd = cargo_bin_cmd!("diffutils"); + cmd.arg("diff"); + if !option.is_empty() { + cmd.arg(option); + } + cmd.arg(file.path()).arg(file.path()); + cmd.assert() + .code(predicate::eq(0)) + .success() + .stdout(predicate::str::is_empty()); + } + Ok(()) + } + + #[test] + fn no_differences_report_identical_files() -> Result<(), Box> { + // same file + let mut file1 = NamedTempFile::new()?; + file1.write_all("foo\n".as_bytes())?; + for option in ["", "-u", "-c", "-e"] { + let mut cmd = cargo_bin_cmd!("diffutils"); + cmd.arg("diff"); + if !option.is_empty() { + cmd.arg(option); + } + cmd.arg("-s").arg(file1.path()).arg(file1.path()); + cmd.assert() + .code(predicate::eq(0)) + .success() + .stdout(predicate::eq(format!( + "Files {} and {} are identical\n", + file1.path().to_string_lossy(), + file1.path().to_string_lossy(), + ))); + } + // two files with the same content + let mut file2 = NamedTempFile::new()?; + file2.write_all("foo\n".as_bytes())?; + for option in ["", "-u", "-c", "-e"] { + let mut cmd = cargo_bin_cmd!("diffutils"); + cmd.arg("diff"); + if !option.is_empty() { + cmd.arg(option); + } + cmd.arg("-s").arg(file1.path()).arg(file2.path()); + cmd.assert() + .code(predicate::eq(0)) + .success() + .stdout(predicate::eq(format!( + "Files {} and {} are identical\n", + file1.path().to_string_lossy(), + file2.path().to_string_lossy(), + ))); + } + Ok(()) + } + + #[test] + fn differences() -> Result<(), Box> { + let mut file1 = NamedTempFile::new()?; + file1.write_all("foo\n".as_bytes())?; + let mut file2 = NamedTempFile::new()?; + file2.write_all("bar\n".as_bytes())?; + for option in ["", "-u", "-c", "-e"] { + let mut cmd = cargo_bin_cmd!("diffutils"); + cmd.arg("diff"); + if !option.is_empty() { + cmd.arg(option); + } + cmd.arg(file1.path()).arg(file2.path()); + cmd.assert() + .code(predicate::eq(1)) + .failure() + .stdout(predicate::str::is_empty().not()); + } + Ok(()) + } + + #[test] + fn differences_brief() -> Result<(), Box> { + let mut file1 = NamedTempFile::new()?; + file1.write_all("foo\n".as_bytes())?; + let mut file2 = NamedTempFile::new()?; + file2.write_all("bar\n".as_bytes())?; + for option in ["", "-u", "-c", "-e"] { + let mut cmd = cargo_bin_cmd!("diffutils"); + cmd.arg("diff"); + if !option.is_empty() { + cmd.arg(option); + } + cmd.arg("-q").arg(file1.path()).arg(file2.path()); + cmd.assert() + .code(predicate::eq(1)) + .failure() + .stdout(predicate::eq(format!( + "Files {} and {} differ\n", + file1.path().to_string_lossy(), + file2.path().to_string_lossy() + ))); + } + Ok(()) + } + + #[test] + fn missing_newline() -> Result<(), Box> { + let mut file1 = NamedTempFile::new()?; + file1.write_all("foo".as_bytes())?; + let mut file2 = NamedTempFile::new()?; + file2.write_all("bar".as_bytes())?; + let mut cmd = cargo_bin_cmd!("diffutils"); + cmd.arg("diff"); + cmd.arg("-e").arg(file1.path()).arg(file2.path()); + cmd.assert() + .code(predicate::eq(2)) + .failure() + .stderr(predicate::str::starts_with("No newline at end of file")); + Ok(()) + } + + #[test] + fn read_from_stdin() -> Result<(), Box> { + let mut file1 = NamedTempFile::new()?; + file1.write_all("foo\n".as_bytes())?; + let mut file2 = NamedTempFile::new()?; + file2.write_all("bar\n".as_bytes())?; + + let mut cmd = cargo_bin_cmd!("diffutils"); + cmd.arg("diff"); + cmd.arg("-u") + .arg(file1.path()) + .arg("-") + .write_stdin("bar\n"); + cmd.assert().code(predicate::eq(1)).failure(); + + let output = cmd.output().unwrap().stdout; + assert_diff_eq!( + output, + format!( + "--- {}\tTIMESTAMP\n+++ -\tTIMESTAMP\n@@ -1 +1 @@\n-foo\n+bar\n", + file1.path().to_string_lossy() + ) + ); + + let mut cmd = cargo_bin_cmd!("diffutils"); + cmd.arg("diff"); + cmd.arg("-u") + .arg("-") + .arg(file2.path()) + .write_stdin("foo\n"); + cmd.assert().code(predicate::eq(1)).failure(); + + let output = cmd.output().unwrap().stdout; + assert_diff_eq!( + output, + format!( + "--- -\tTIMESTAMP\n+++ {}\tTIMESTAMP\n@@ -1 +1 @@\n-foo\n+bar\n", + file2.path().to_string_lossy() + ) + ); + + let mut cmd = cargo_bin_cmd!("diffutils"); + cmd.arg("diff"); + cmd.arg("-u").arg("-").arg("-"); + cmd.assert() + .code(predicate::eq(0)) + .success() + .stdout(predicate::str::is_empty()); + + #[cfg(unix)] + { + let mut cmd = cargo_bin_cmd!("diffutils"); + cmd.arg("diff"); + cmd.arg("-u") + .arg(file1.path()) + .arg("/dev/stdin") + .write_stdin("bar\n"); + cmd.assert().code(predicate::eq(1)).failure(); + + let output = cmd.output().unwrap().stdout; + assert_diff_eq!( + output, + format!( + "--- {}\tTIMESTAMP\n+++ /dev/stdin\tTIMESTAMP\n@@ -1 +1 @@\n-foo\n+bar\n", + file1.path().to_string_lossy() + ) + ); + } + + Ok(()) + } + + #[test] + fn compare_file_to_directory() -> Result<(), Box> { + let tmp_dir = tempdir()?; + + let directory = tmp_dir.path().join("d"); + let _ = std::fs::create_dir(&directory); + + let a_path = tmp_dir.path().join("a"); + let mut a = File::create(&a_path).unwrap(); + a.write_all(b"a\n").unwrap(); + + let da_path = directory.join("a"); + let mut da = File::create(&da_path).unwrap(); + da.write_all(b"da\n").unwrap(); + + let mut cmd = cargo_bin_cmd!("diffutils"); + cmd.arg("diff"); + cmd.arg("-u").arg(&directory).arg(&a_path); + cmd.assert().code(predicate::eq(1)).failure(); + + let output = cmd.output().unwrap().stdout; + assert_diff_eq!( + output, + format!( + "--- {}\tTIMESTAMP\n+++ {}\tTIMESTAMP\n@@ -1 +1 @@\n-da\n+a\n", + da_path.display(), + a_path.display() + ) + ); + + let mut cmd = cargo_bin_cmd!("diffutils"); + cmd.arg("diff"); + cmd.arg("-u").arg(&a_path).arg(&directory); + cmd.assert().code(predicate::eq(1)).failure(); + + let output = cmd.output().unwrap().stdout; + assert_diff_eq!( + output, + format!( + "--- {}\tTIMESTAMP\n+++ {}\tTIMESTAMP\n@@ -1 +1 @@\n-a\n+da\n", + a_path.display(), + da_path.display() + ) + ); + + Ok(()) + } + + // Regression for #269: the `-y` gutter marker must sit at the middle of the gutter. + #[test] + fn sdiff_gutter_marker_column() -> Result<(), Box> { + let mut file1 = NamedTempFile::new()?; + file1.write_all("aa\n".as_bytes())?; + let mut file2 = NamedTempFile::new()?; + file2.write_all("bb\n".as_bytes())?; + + // --expand-tabs: padding is spaces. + let mut cmd = cargo_bin_cmd!("diffutils"); + cmd.arg("diff") + .arg("-y") + .arg("--width=40") + .arg("--expand-tabs") + .arg(file1.path()) + .arg(file2.path()); + let output = cmd.output().unwrap().stdout; + assert_eq!( + output, + b"aa <\n > bb\n" + ); + + // Default (expand_tabs: false): padding goes through the tab path. + let mut cmd = cargo_bin_cmd!("diffutils"); + cmd.arg("diff") + .arg("-y") + .arg("--width=40") + .arg(file1.path()) + .arg(file2.path()); + let output = cmd.output().unwrap().stdout; + assert_eq!(output, b"aa\t\t <\n\t\t >\tbb\n"); + + Ok(()) + } +} + +mod cmp { + use super::*; + + // A file whose metadata length is 0 but which still yields bytes (/dev/zero) + // collapses the offset column to two characters, so the padding subtraction + // underflowed once the running offset reached three digits. + #[test] + #[cfg(unix)] + fn cmp_verbose_zero_length_metadata() -> Result<(), Box> { + let mut file = NamedTempFile::new()?; + file.write_all(&[0xffu8; 150])?; + + let mut cmd = cargo_bin_cmd!("diffutils"); + cmd.arg("cmp") + .arg("--verbose") + .arg("/dev/zero") + .arg(file.path()); + cmd.assert() + .code(predicate::eq(1)) + .stdout(predicate::str::contains("150 0 377")); + Ok(()) + } + + #[test] + fn cmp_incompatible_params() -> Result<(), Box> { + let mut cmd = cargo_bin_cmd!("diffutils"); + cmd.arg("cmp"); + cmd.arg("-l"); + cmd.arg("-s"); + cmd.arg("/etc/passwd").arg("/etc/group"); + cmd.assert() + .code(predicate::eq(2)) + .failure() + .stderr(predicate::str::ends_with( + ": options -l and -s are incompatible\n", + )); + + Ok(()) + } + + #[test] + fn cmp_stdin() -> Result<(), Box> { + let tmp_dir = tempdir()?; + + let a_path = tmp_dir.path().join("a"); + let mut a = File::create(&a_path).unwrap(); + a.write_all(b"a\n").unwrap(); + + let mut cmd = cargo_bin_cmd!("diffutils"); + cmd.arg("cmp"); + cmd.arg(&a_path); + cmd.write_stdin("a\n"); + cmd.assert() + .code(predicate::eq(0)) + .success() + .stderr(predicate::str::is_empty()) + .stdout(predicate::str::is_empty()); + + let mut cmd = cargo_bin_cmd!("diffutils"); + cmd.env("LC_ALL", "C"); + cmd.arg("cmp"); + cmd.arg(&a_path); + cmd.write_stdin("b\n"); + cmd.assert() + .code(predicate::eq(1)) + .failure() + .stderr(predicate::str::is_empty()) + .stdout(predicate::str::ends_with(" - differ: char 1, line 1\n")); + + Ok(()) + } + + #[test] + fn cmp_equal_files() -> Result<(), Box> { + let tmp_dir = tempdir()?; + + let a_path = tmp_dir.path().join("a"); + let mut a = File::create(&a_path).unwrap(); + a.write_all(b"a\n").unwrap(); + + let b_path = tmp_dir.path().join("b"); + let mut b = File::create(&b_path).unwrap(); + b.write_all(b"a\n").unwrap(); + + let mut cmd = cargo_bin_cmd!("diffutils"); + cmd.arg("cmp"); + cmd.arg(&a_path).arg(&b_path); + cmd.assert() + .code(predicate::eq(0)) + .success() + .stderr(predicate::str::is_empty()) + .stdout(predicate::str::is_empty()); + + Ok(()) + } + + #[test] + fn cmp_one_file_empty() -> Result<(), Box> { + let tmp_dir = tempdir()?; + + let a_path = tmp_dir.path().join("a"); + let mut a = File::create(&a_path).unwrap(); + a.write_all(b"a\n").unwrap(); + + let b_path = tmp_dir.path().join("b"); + let _ = File::create(&b_path).unwrap(); + + let mut cmd = cargo_bin_cmd!("diffutils"); + cmd.arg("cmp"); + cmd.arg(&a_path).arg(&b_path); + cmd.assert() + .code(predicate::eq(1)) + .failure() + .stderr(predicate::str::contains(" EOF on ")) + .stderr(predicate::str::ends_with(" which is empty\n")); + + Ok(()) + } + + #[test] + fn cmp_immediate_difference() -> Result<(), Box> { + let tmp_dir = tempdir()?; + + let a_path = tmp_dir.path().join("a"); + let mut a = File::create(&a_path).unwrap(); + a.write_all(b"abc\n").unwrap(); + + let b_path = tmp_dir.path().join("b"); + let mut b = File::create(&b_path).unwrap(); + b.write_all(b"bcd\n").unwrap(); + + let mut cmd = cargo_bin_cmd!("diffutils"); + cmd.env("LC_ALL", "C"); + cmd.arg("cmp"); + cmd.arg(&a_path).arg(&b_path); + cmd.assert() + .code(predicate::eq(1)) + .failure() + .stdout(predicate::str::ends_with(" differ: char 1, line 1\n")); + + let mut cmd = cargo_bin_cmd!("diffutils"); + cmd.env("LC_ALL", "C"); + cmd.arg("cmp"); + cmd.arg("-b"); + cmd.arg(&a_path).arg(&b_path); + cmd.assert() + .code(predicate::eq(1)) + .failure() + .stderr(predicate::str::is_empty()) + .stdout(predicate::str::ends_with( + " differ: byte 1, line 1 is 141 a 142 b\n", + )); + + let mut cmd = cargo_bin_cmd!("diffutils"); + cmd.env("LC_ALL", "C"); + cmd.arg("cmp"); + cmd.arg("-l"); + cmd.arg(&a_path).arg(&b_path); + cmd.assert() + .code(predicate::eq(1)) + .failure() + .stderr(predicate::str::is_empty()) + .stdout(predicate::eq("1 141 142\n2 142 143\n3 143 144\n")); + + let mut cmd = cargo_bin_cmd!("diffutils"); + cmd.env("LC_ALL", "C"); + cmd.arg("cmp"); + cmd.arg("-l"); + cmd.arg("-b"); + cmd.arg(&a_path).arg(&b_path); + cmd.assert() + .code(predicate::eq(1)) + .failure() + .stderr(predicate::str::is_empty()) + .stdout(predicate::eq( + "1 141 a 142 b\n2 142 b 143 c\n3 143 c 144 d\n", + )); + + Ok(()) + } + + #[test] + fn cmp_newline_difference() -> Result<(), Box> { + let tmp_dir = tempdir()?; + + let a_path = tmp_dir.path().join("a"); + let mut a = File::create(&a_path).unwrap(); + a.write_all(b"abc\ndefg").unwrap(); + + let b_path = tmp_dir.path().join("b"); + let mut b = File::create(&b_path).unwrap(); + b.write_all(b"abc\ndef\ng").unwrap(); + + let mut cmd = cargo_bin_cmd!("diffutils"); + cmd.env("LC_ALL", "C"); + cmd.arg("cmp"); + cmd.arg(&a_path).arg(&b_path); + cmd.assert() + .code(predicate::eq(1)) + .failure() + .stderr(predicate::str::is_empty()) + .stdout(predicate::str::ends_with(" differ: char 8, line 2\n")); + + let mut cmd = cargo_bin_cmd!("diffutils"); + cmd.env("LC_ALL", "C"); + cmd.arg("cmp"); + cmd.arg("-b"); + cmd.arg(&a_path).arg(&b_path); + cmd.assert() + .code(predicate::eq(1)) + .failure() + .stderr(predicate::str::is_empty()) + .stdout(predicate::str::ends_with( + " differ: byte 8, line 2 is 147 g 12 ^J\n", + )); + + let mut cmd = cargo_bin_cmd!("diffutils"); + cmd.env("LC_ALL", "C"); + cmd.arg("cmp"); + cmd.arg("-l"); + cmd.arg(&a_path).arg(&b_path); + cmd.assert() + .code(predicate::eq(1)) + .failure() + .stdout(predicate::str::starts_with("8 147 12\n")) + .stderr(predicate::str::contains(" EOF on")) + .stderr(predicate::str::ends_with(" after byte 8\n")); + + let mut cmd = cargo_bin_cmd!("diffutils"); + cmd.env("LC_ALL", "C"); + cmd.arg("cmp"); + cmd.arg("-b"); + cmd.arg("-l"); + cmd.arg(&a_path).arg(&b_path); + cmd.assert() + .code(predicate::eq(1)) + .failure() + .stdout(predicate::str::starts_with("8 147 g 12 ^J\n")) + .stderr(predicate::str::contains(" EOF on")) + .stderr(predicate::str::ends_with(" after byte 8\n")); + + Ok(()) + } + + #[test] + fn cmp_max_bytes() -> Result<(), Box> { + let tmp_dir = tempdir()?; + + let a_path = tmp_dir.path().join("a"); + let mut a = File::create(&a_path).unwrap(); + a.write_all(b"abc efg ijkl\n").unwrap(); + + let b_path = tmp_dir.path().join("b"); + let mut b = File::create(&b_path).unwrap(); + b.write_all(b"abcdefghijkl\n").unwrap(); + + let mut cmd = cargo_bin_cmd!("diffutils"); + cmd.arg("cmp"); + cmd.arg("-l"); + cmd.arg("-b"); + cmd.arg("-n"); + cmd.arg("3"); + cmd.arg(&a_path).arg(&b_path); + cmd.assert() + .code(predicate::eq(0)) + .success() + .stderr(predicate::str::is_empty()) + .stdout(predicate::str::is_empty()); + + let mut cmd = cargo_bin_cmd!("diffutils"); + cmd.arg("cmp"); + cmd.arg("-l"); + cmd.arg("-b"); + cmd.arg("-n"); + cmd.arg("4"); + cmd.arg(&a_path).arg(&b_path); + cmd.assert() + .code(predicate::eq(1)) + .failure() + .stderr(predicate::str::is_empty()) + .stdout(predicate::eq("4 40 144 d\n")); + + let mut cmd = cargo_bin_cmd!("diffutils"); + cmd.arg("cmp"); + cmd.arg("-l"); + cmd.arg("-b"); + cmd.arg("-n"); + cmd.arg("13"); + cmd.arg(&a_path).arg(&b_path); + cmd.assert() + .code(predicate::eq(1)) + .failure() + .stderr(predicate::str::is_empty()) + .stdout(predicate::eq(" 4 40 144 d\n 8 40 150 h\n")); + Ok(()) + } + + #[test] + fn cmp_skip_args_parsing() -> Result<(), Box> { + let tmp_dir = tempdir()?; + + let a_path = tmp_dir.path().join("a"); + let mut a = File::create(&a_path).unwrap(); + a.write_all(b"---abc\n").unwrap(); + + let b_path = tmp_dir.path().join("b"); + let mut b = File::create(&b_path).unwrap(); + b.write_all(b"###abc\n").unwrap(); + + let mut cmd = cargo_bin_cmd!("diffutils"); + cmd.env("LC_ALL", "C"); + cmd.arg("cmp"); + cmd.arg("-i"); + cmd.arg("3"); + cmd.arg(&a_path).arg(&b_path); + cmd.assert() + .code(predicate::eq(0)) + .success() + .stderr(predicate::str::is_empty()) + .stdout(predicate::str::is_empty()); + + // Positional skips should be ignored + let mut cmd = cargo_bin_cmd!("diffutils"); + cmd.env("LC_ALL", "C"); + cmd.arg("cmp"); + cmd.arg("-i"); + cmd.arg("3"); + cmd.arg(&a_path).arg(&b_path); + cmd.arg("1").arg("1"); + cmd.assert() + .code(predicate::eq(0)) + .success() + .stderr(predicate::str::is_empty()) + .stdout(predicate::str::is_empty()); + + // Single positional argument should only affect first file. + let mut cmd = cargo_bin_cmd!("diffutils"); + cmd.env("LC_ALL", "C"); + cmd.arg("cmp"); + cmd.arg(&a_path).arg(&b_path); + cmd.arg("3"); + cmd.assert() + .code(predicate::eq(1)) + .failure() + .stderr(predicate::str::is_empty()) + .stdout(predicate::str::ends_with(" differ: char 1, line 1\n")); + + let mut cmd = cargo_bin_cmd!("diffutils"); + cmd.env("LC_ALL", "C"); + cmd.arg("cmp"); + cmd.arg(&a_path).arg(&b_path); + cmd.arg("3"); + cmd.arg("3"); + cmd.assert() + .code(predicate::eq(0)) + .success() + .stderr(predicate::str::is_empty()) + .stdout(predicate::str::is_empty()); + + Ok(()) + } + + #[test] + fn cmp_skip_suffix_parsing() -> Result<(), Box> { + let tmp_dir = tempdir()?; + + let a_path = tmp_dir.path().join("a"); + let mut a = File::create(&a_path).unwrap(); + writeln!(a, "{}c", "a".repeat(1024)).unwrap(); + a.flush().unwrap(); + + let b_path = tmp_dir.path().join("b"); + let mut b = File::create(&b_path).unwrap(); + writeln!(b, "{}c", "b".repeat(1024)).unwrap(); + b.flush().unwrap(); + + let mut cmd = cargo_bin_cmd!("diffutils"); + cmd.arg("cmp"); + cmd.arg("--ignore-initial=1K"); + cmd.arg(&a_path).arg(&b_path); + cmd.assert() + .code(predicate::eq(0)) + .success() + .stderr(predicate::str::is_empty()) + .stdout(predicate::str::is_empty()); + + Ok(()) + } + + #[test] + fn cmp_skip() -> Result<(), Box> { + let tmp_dir = tempdir()?; + + let a_path = tmp_dir.path().join("a"); + let mut a = File::create(&a_path).unwrap(); + a.write_all(b"abc efg ijkl\n").unwrap(); + + let b_path = tmp_dir.path().join("b"); + let mut b = File::create(&b_path).unwrap(); + b.write_all(b"abcdefghijkl\n").unwrap(); + + let mut cmd = cargo_bin_cmd!("diffutils"); + cmd.arg("cmp"); + cmd.arg("-l"); + cmd.arg("-b"); + cmd.arg("-i"); + cmd.arg("8"); + cmd.arg(&a_path).arg(&b_path); + cmd.assert() + .code(predicate::eq(0)) + .success() + .stderr(predicate::str::is_empty()) + .stdout(predicate::str::is_empty()); + + let mut cmd = cargo_bin_cmd!("diffutils"); + cmd.arg("cmp"); + cmd.arg("-b"); + cmd.arg("-i"); + cmd.arg("4"); + cmd.arg(&a_path).arg(&b_path); + cmd.assert() + .code(predicate::eq(1)) + .failure() + .stderr(predicate::str::is_empty()) + .stdout(predicate::str::ends_with( + " differ: byte 4, line 1 is 40 150 h\n", + )); + + Ok(()) + } + + #[test] + fn cmp_binary() -> Result<(), Box> { + let tmp_dir = tempdir()?; + + let mut bytes = vec![0, 15, 31, 32, 33, 40, 64, 126, 127, 128, 129, 200, 254, 255]; + + let a_path = tmp_dir.path().join("a"); + let mut a = File::create(&a_path).unwrap(); + a.write_all(&bytes).unwrap(); + + bytes.reverse(); + + let b_path = tmp_dir.path().join("b"); + let mut b = File::create(&b_path).unwrap(); + b.write_all(&bytes).unwrap(); + + let mut cmd = cargo_bin_cmd!("diffutils"); + cmd.arg("cmp"); + cmd.arg("-l"); + cmd.arg("-b"); + cmd.arg(&a_path).arg(&b_path); + cmd.assert() + .code(predicate::eq(1)) + .failure() + .stdout(predicate::eq(concat!( + " 1 0 ^@ 377 M-^?\n", + " 2 17 ^O 376 M-~\n", + " 3 37 ^_ 310 M-H\n", + " 4 40 201 M-^A\n", + " 5 41 ! 200 M-^@\n", + " 6 50 ( 177 ^?\n", + " 7 100 @ 176 ~\n", + " 8 176 ~ 100 @\n", + " 9 177 ^? 50 (\n", + "10 200 M-^@ 41 !\n", + "11 201 M-^A 40 \n", + "12 310 M-H 37 ^_\n", + "13 376 M-~ 17 ^O\n", + "14 377 M-^? 0 ^@\n" + ))); + + Ok(()) + } + + #[test] + #[cfg(not(windows))] + fn cmp_fast_paths() -> Result<(), Box> { + let tmp_dir = tempdir()?; + + // This test mimics one found in the GNU cmp test suite. It is used for + // validating the /dev/null optimization. + let a_path = tmp_dir.path().join("a"); + let a = File::create(&a_path).unwrap(); + a.set_len(14 * 1024 * 1024 * 1024 * 1024).unwrap(); + + let b_path = tmp_dir.path().join("b"); + let b = File::create(&b_path).unwrap(); + b.set_len(15 * 1024 * 1024 * 1024 * 1024).unwrap(); + + let dev_null = OpenOptions::new().write(true).open("/dev/null").unwrap(); + + let mut child = std::process::Command::new(assert_cmd::cargo::cargo_bin!("diffutils")) + .arg("cmp") + .arg(&a_path) + .arg(&b_path) + .stdout(dev_null) + .spawn() + .unwrap(); + + // Bound the runtime to a very short time that still allows for some resource + // constraint to slow it down while also allowing very fast systems to exit as + // early as possible. + const MAX_TRIES: u8 = 50; + for tries in 0..=MAX_TRIES { + if tries == MAX_TRIES { + panic!("cmp took too long to run, /dev/null optimization probably not working") + } + match child.try_wait() { + Ok(Some(status)) => { + assert_eq!(status.code(), Some(1)); + break; + } + Ok(None) => (), + Err(e) => panic!("{e:#?}"), + } + std::thread::sleep(std::time::Duration::from_millis(10)); + } + + // Two stdins should be equal + let mut cmd = cargo_bin_cmd!("diffutils"); + cmd.arg("cmp"); + cmd.arg("-"); + cmd.arg("-"); + cmd.assert() + .code(predicate::eq(0)) + .success() + .stdout(predicate::str::is_empty()) + .stderr(predicate::str::is_empty()); + + // Files with longer than block size equal segments should still report + // the correct line number for the difference. Assumes 8KB block size (see + // https://github.com/rust-lang/rust/blob/master/library/std/src/sys_common/io.rs), + // create a 24KB equality. + let mut bytes = " ".repeat(4095); + bytes.push('\n'); + bytes.push_str(&" ".repeat(4096)); + + let bytes = bytes.repeat(3); + let bytes = bytes.as_bytes(); + + let a_path = tmp_dir.path().join("a"); + let mut a = File::create(&a_path).unwrap(); + a.write_all(bytes).unwrap(); + a.write_all(b"A").unwrap(); + + let b_path = tmp_dir.path().join("b"); + let mut b = File::create(&b_path).unwrap(); + b.write_all(bytes).unwrap(); + b.write_all(b"B").unwrap(); + + let mut cmd = cargo_bin_cmd!("diffutils"); + cmd.arg("cmp"); + cmd.arg(&a_path).arg(&b_path); + cmd.env("LC_ALL", "en_US"); + cmd.assert() + .code(predicate::eq(1)) + .failure() + .stdout(predicate::str::ends_with(" differ: byte 24577, line 4\n")); + + Ok(()) + } +} diff --git a/userland/diffutils/tests/print-test-results.sh b/userland/diffutils/tests/print-test-results.sh new file mode 100755 index 0000000..23136d6 --- /dev/null +++ b/userland/diffutils/tests/print-test-results.sh @@ -0,0 +1,38 @@ +#!/bin/bash + +# Print the test results written to a JSON file by run-upstream-testsuite.sh +# in a markdown format. The printout includes the name of the test, the result, +# the URL to the test script and the contents of stdout and stderr. +# It can be used verbatim as the description when filing an issue for a test +# with an unexpected result. + +json="test-results.json" +[[ -n $1 ]] && json="$1" + +codeblock () { echo -e "\`\`\`\n$1\n\`\`\`"; } + +jq -c '.tests[]' "$json" | while read -r test +do + name=$(echo "$test" | jq -r '.test') + echo "# test: $name" + result=$(echo "$test" | jq -r '.result') + echo "result: $result" + url=$(echo "$test" | jq -r '.url') + echo "url: $url" + if [[ "$result" != "SKIP" ]] + then + stdout=$(echo "$test" | jq -r '.stdout' | base64 -d) + if [[ -n "$stdout" ]] + then + echo "## stdout" + codeblock "$stdout" + fi + stderr=$(echo "$test" | jq -r '.stderr' | base64 -d) + if [[ -n "$stderr" ]] + then + echo "## stderr" + codeblock "$stderr" + fi + fi + echo "" +done diff --git a/userland/diffutils/tests/run-upstream-testsuite.sh b/userland/diffutils/tests/run-upstream-testsuite.sh new file mode 100755 index 0000000..39de1c8 --- /dev/null +++ b/userland/diffutils/tests/run-upstream-testsuite.sh @@ -0,0 +1,168 @@ +#!/bin/bash + +# Run the GNU upstream test suite for diffutils against a local build of the +# Rust implementation, print out a summary of the test results, and writes a +# JSON file ('test-results.json') containing detailed information about the +# test run. + +# The JSON file contains metadata about the test run, and for each test the +# result as well as the contents of stdout, stderr, and of all the files +# written by the test script, if any (excluding subdirectories). + +# The script takes a shortcut to fetch only the test suite from the upstream +# repository and carefully avoids running the autotools machinery which is +# time-consuming and resource-intensive, and doesn't offer the option to not +# build the upstream binaries. As a consequence, the environment in which the +# tests are run might not match exactly that used when the upstream tests are +# run through the autotools. + +# Exit codes: 0 if all tests passed, 1 if at least one test failed, and 2 if +# the test suite could not be run at all (e.g. the upstream repository couldn't +# be fetched). Callers must treat 2 as an infrastructure error: no meaningful +# result was produced. + +# By default it expects a release build of the diffutils binary, but a +# different build profile can be specified as an argument +# (e.g. 'dev' or 'test'). +# Unless overridden by the $TESTS environment variable, all tests in the test +# suite will be run. Tests targeting a command that is not yet implemented +# (e.g. diff3 or sdiff) are skipped. + +scriptpath=$(dirname "$(readlink -f "$0")") +rev=$(git rev-parse HEAD) +gnu_version="3.12" +gnu_tarball_blake2b=5b4593b39da71578d7f975603abe9359be215b9ac76548a6ab0d6e3838bb103c7ffcddf7fa01abcd5c6289db9a2f16b43aa3d5e846a9aa4b8db866763c2660de + +# Report an infrastructure error: the test suite could not be run at all +die() { + echo "ERROR: $*" >&2 + exit 2 +} + +# Allow passing a specific profile as parameter (default to "release") +profile="release" +[[ -n $1 ]] && profile="$1" + +# Verify that the diffutils binary was built for the requested profile +binary="$scriptpath/../target/$profile/diffutils" +if [[ ! -x "$binary" ]] +then + die "Missing build for profile $profile" +fi + +# Work in a temporary directory +tempdir=$(mktemp -d) +trap 'rm -rf "$tempdir"' EXIT +cd "$tempdir" || die "Cannot enter temporary directory $tempdir" + +# Fetch the upstream test suite +echo "Fetching upstream test suite" +curl -fsSL --retry 5 https://ftpmirror.gnu.org/diffutils/diffutils-${gnu_version}.tar.xz -o gnu-diffutils.tar.xz || die "Failed to fetch test suite" +echo ${gnu_tarball_blake2b} gnu-diffutils.tar.xz \ + | b2sum --check || die "Failed to verify checksum of tarball of GNU diffutils" +mkdir -p diffutils +tar xJf gnu-diffutils.tar.xz --strip-components=1 -C diffutils || die "Failed to extract test suite" + +cd diffutils || die "Test suite src is broken" + +# Ensure that calling `diff` invokes the built `diffutils` binary instead of +# the upstream `diff` binary that is most likely installed on the system +cd src || die "Missing the directory holding the diff and cmp symlinks" +ln -s "$binary" diff +ln -s "$binary" cmp +cd ../tests || die "Cannot enter the upstream tests directory" + +if [[ -n "$TESTS" ]] +then + tests="$TESTS" +else + # Get a list of all upstream tests (default if $TESTS isn't set) + echo -e '\n\nprinttests:\n\t@echo "${TESTS}"' >> Makefile.am + tests=$(make -f Makefile.am printtests) +fi +total=$(echo "$tests" | wc -w) +(( total > 0 )) || die "No test to run: the upstream test list is empty" +echo "Running $total tests" +export LC_ALL=C +export KEEP=yes +timestamp=$(date -Iseconds) +passed=0 +failed=0 +skipped=0 +normal="$(tput sgr0)" +for test in $tests +do + result="FAIL" + # Run only the tests that invoke `diff` or `cmp`, + # because other binaries aren't implemented yet + if ! grep -E -s -q "(diff3|sdiff)" "$test" + then + sh "$test" 1> stdout.txt 2> stderr.txt && result="PASS" + if [[ $? = 77 ]] + then + result="SKIP" + else + json+="{\"test\":\"$test\",\"result\":\"$result\"," + json+="\"stdout\":\"$(base64 -w0 < stdout.txt)\"," + json+="\"stderr\":\"$(base64 -w0 < stderr.txt)\"," + json+="\"files\":{" + cd gt-$test.* + # Note: this doesn't include the contents of subdirectories, + # but there isn't much value added in doing so + for file in * + do + # Encode the name with jq: some tests create files whose name contains + # quotes or control characters, which would produce invalid JSON + [[ -f "$file" ]] && json+="$(jq -Rn --arg name "$file" '$name'):\"$(base64 -w0 < "$file")\"," + done + json="${json%,}}}," + cd - > /dev/null + [[ "$result" = "PASS" ]] && (( passed++ )) + [[ "$result" = "FAIL" ]] && (( failed++ )) + fi + else + result="SKIP" + fi + color=2 # green + [[ "$result" = "FAIL" ]] && color=1 # red + if [[ $result = "SKIP" ]] + then + (( skipped++ )) + json+="{\"test\":\"$test\",\"result\":\"$result\"}," + color=3 # yellow + fi + printf " %-40s $(tput setaf $color)$result$(tput sgr0)\n" "$test" +done +echo "" +echo -n "Summary: TOTAL: $total / " +echo -n "$(tput setaf 2)PASS$normal: $passed / " +echo -n "$(tput setaf 1)FAIL$normal: $failed / " +echo "$(tput setaf 3)SKIP$normal: $skipped" +echo "" + +json="\"tests\":[${json%,}]" +metadata="\"timestamp\":\"$timestamp\"," +metadata+="\"revision\":\"$rev\"," +metadata+="\"upstream-version\":\"$gnu_version\"," +if [[ -n "$GITHUB_ACTIONS" ]] +then + metadata+="\"branch\":\"$GITHUB_REF\"," +fi +json="{$metadata $json}" + +# Clean up +cd "$scriptpath" + +# Write the results out only once they are known to be valid JSON, so that a +# malformed (or truncated) file is never left behind for the caller to consume +resultsfile="test-results.json" +if ! echo "$json" | jq > "$resultsfile.tmp" +then + rm -f "$resultsfile.tmp" + die "Generated invalid JSON results" +fi +mv "$resultsfile.tmp" "$resultsfile" +echo "Results written to $scriptpath/$resultsfile" + +(( failed > 0 )) && exit 1 +exit 0 diff --git a/userland/diffutils/tools.txt b/userland/diffutils/tools.txt new file mode 100644 index 0000000..8b974e4 --- /dev/null +++ b/userland/diffutils/tools.txt @@ -0,0 +1,2 @@ +diff +cmp diff --git a/userland/diffutils/util/compare_test_results.py b/userland/diffutils/util/compare_test_results.py new file mode 100755 index 0000000..00469d6 --- /dev/null +++ b/userland/diffutils/util/compare_test_results.py @@ -0,0 +1,162 @@ +#!/usr/bin/env python3 + +""" +Compare the current GNU test results to the last results gathered from the main branch to +highlight if a PR is making the results better/worse. +Don't exit with error code if all failing tests are in the ignore-intermittent.txt list. + +Exit status: 0 if the comparison shows no new failure, 1 if new non-intermittent +failures appeared, and 2 if the comparison couldn't be performed at all (e.g. one +of the result files is missing or malformed). +""" + +import json +import sys +import argparse +from pathlib import Path + + +def load_ignore_list(ignore_file): + """Load list of intermittent test names to ignore from file.""" + ignore_set = set() + if ignore_file and Path(ignore_file).exists(): + with open(ignore_file, "r") as f: + for line in f: + line = line.strip() + if line and not line.startswith("#"): + ignore_set.add(line) + return ignore_set + + +def extract_test_results(json_data): + """Extract test results from a diffutils test-results.json. + + Note: unlike sed, diffutils JSON has no 'summary' object — results are + computed from the 'tests' array using the 'result' and 'test' fields. + """ + tests = json_data.get("tests", []) + passed = sum(1 for t in tests if t.get("result") == "PASS") + failed = sum(1 for t in tests if t.get("result") == "FAIL") + skipped = sum(1 for t in tests if t.get("result") == "SKIP") + summary = {"total": len(tests), "passed": passed, "failed": failed, "skipped": skipped} + failed_tests = [t["test"] for t in tests if t.get("result") == "FAIL"] + return summary, failed_tests + + +def compare_results(current_file, reference_file, ignore_file=None, output_file=None): + """Compare current results with reference results.""" + ignore_set = load_ignore_list(ignore_file) + + try: + with open(current_file, "r") as f: + current_data = json.load(f) + current_summary, current_failed = extract_test_results(current_data) + except Exception as e: + print(f"Error loading current results: {e}") + return 2 + + try: + with open(reference_file, "r") as f: + reference_data = json.load(f) + reference_summary, reference_failed = extract_test_results(reference_data) + except Exception as e: + print(f"Error loading reference results: {e}") + return 2 + + # Calculate differences + pass_diff = int(current_summary.get("passed", 0)) - int(reference_summary.get("passed", 0)) + fail_diff = int(current_summary.get("failed", 0)) - int(reference_summary.get("failed", 0)) + total_diff = int(current_summary.get("total", 0)) - int(reference_summary.get("total", 0)) + + # Find new failures and improvements + current_failed_set = set(current_failed) + reference_failed_set = set(reference_failed) + + new_failures = current_failed_set - reference_failed_set + improvements = reference_failed_set - current_failed_set + + # Filter out intermittent failures + non_intermittent_new_failures = new_failures - ignore_set + + # Check if results are identical (no changes) + no_changes = ( + pass_diff == 0 + and fail_diff == 0 + and total_diff == 0 + and not new_failures + and not improvements + ) + + # If no changes, write empty output to prevent comment posting + if no_changes: + if output_file: + with open(output_file, "w") as f: + f.write("") + return 0 + + # Prepare output message + output_lines = [] + + output_lines.append("Test results comparison:") + output_lines.append( + f" Current: TOTAL: {current_summary.get('total', 0)} / PASSED: {current_summary.get('passed', 0)} / FAILED: {current_summary.get('failed', 0)} / SKIPPED: {current_summary.get('skipped', 0)}" + ) + output_lines.append( + f" Reference: TOTAL: {reference_summary.get('total', 0)} / PASSED: {reference_summary.get('passed', 0)} / FAILED: {reference_summary.get('failed', 0)} / SKIPPED: {reference_summary.get('skipped', 0)}" + ) + output_lines.append("") + + if pass_diff != 0 or fail_diff != 0 or total_diff != 0: + output_lines.append("Changes from main branch:") + output_lines.append(f" TOTAL: {total_diff:+d}") + output_lines.append(f" PASSED: {pass_diff:+d}") + output_lines.append(f" FAILED: {fail_diff:+d}") + output_lines.append("") + + if new_failures: + output_lines.append(f"New test failures ({len(new_failures)}):") + for test in sorted(new_failures): + if test in ignore_set: + output_lines.append(f" - {test} (intermittent)") + else: + output_lines.append(f" - {test}") + output_lines.append("") + + if improvements: + output_lines.append(f"Test improvements ({len(improvements)}):") + for test in sorted(improvements): + output_lines.append(f" + {test}") + output_lines.append("") + + output_text = "\n".join(output_lines) + if output_file: + with open(output_file, "w") as f: + f.write(output_text) + else: + print(output_text) + + if non_intermittent_new_failures: + print( + f"ERROR: Found {len(non_intermittent_new_failures)} new non-intermittent test failures" + ) + return 1 + + return 0 + + +def main(): + parser = argparse.ArgumentParser(description="Compare GNU diffutils test results") + parser.add_argument("current", help="Current test results JSON file") + parser.add_argument("reference", help="Reference test results JSON file") + parser.add_argument( + "--ignore-file", help="File containing intermittent test names to ignore" + ) + parser.add_argument("--output", help="Output file for comparison results") + + args = parser.parse_args() + + return compare_results(args.current, args.reference, args.ignore_file, args.output) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/userland/exec-tools/Cargo.toml b/userland/exec-tools/Cargo.toml new file mode 100644 index 0000000..b073a14 --- /dev/null +++ b/userland/exec-tools/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "sausage-exec-tools" +version = "0.1.0" +edition = "2024" + +[[bin]] +name = "env" +path = "src/env.rs" + +[[bin]] +name = "timeout" +path = "src/timeout.rs" + +[dependencies] +sausage-spawn = { path = "../spawn" } diff --git a/userland/exec-tools/src/env.rs b/userland/exec-tools/src/env.rs new file mode 100644 index 0000000..19c4857 --- /dev/null +++ b/userland/exec-tools/src/env.rs @@ -0,0 +1,20 @@ +//! Print the environment or start a command with environment overrides. +use sausage_spawn::Command; +fn run() -> Result { + let mut args = std::env::args().skip(1).peekable(); + let mut clear = false; + if args.peek().is_some_and(|a| a == "-i" || a == "--ignore-environment") { clear = true; args.next(); } + let mut values = if clear { std::collections::BTreeMap::new() } else { std::env::vars().collect() }; + while args.peek().is_some_and(|a| a.contains('=')) { + let arg = args.next().unwrap(); let (key, value) = arg.split_once('=').unwrap(); + values.insert(key.to_owned(), value.to_owned()); + } + if let Some(program) = args.next() { + let mut command = Command::new(program); command.env_clear(); + for (key, value) in values { command.env(key, value); } + command.args(args).status() + } else { + for (key, value) in values { println!("{key}={value}"); } Ok(0) + } +} +fn main() { std::process::exit(match run() { Ok(code) => code, Err(e) => { eprintln!("env: {e}"); if e.kind() == std::io::ErrorKind::NotFound { 127 } else { 126 } } }); } diff --git a/userland/exec-tools/src/timeout.rs b/userland/exec-tools/src/timeout.rs new file mode 100644 index 0000000..c2f48dc --- /dev/null +++ b/userland/exec-tools/src/timeout.rs @@ -0,0 +1,25 @@ +//! Stop a world child after a deadline. Expiration returns status 124. +use sausage_spawn::Command; +use std::time::{Duration, Instant}; +fn run() -> Result> { + let mut args = std::env::args().skip(1); + let duration = args.next().ok_or("a duration is required")?; + let (number, multiplier) = match duration.as_bytes().last() { + Some(b's') => (&duration[..duration.len()-1], 1.0), + Some(b'm') => (&duration[..duration.len()-1], 60.0), + Some(b'h') => (&duration[..duration.len()-1], 3600.0), + Some(b'd') => (&duration[..duration.len()-1], 86400.0), + _ => (&duration[..], 1.0), + }; + let duration = Duration::try_from_secs_f64(number.parse::()? * multiplier)?; + let program = args.next().ok_or("a command is required")?; + let child = Command::new(program).args(args).new_group().spawn()?; + if duration.is_zero() { return Ok(child.wait()); } + let deadline = Instant::now() + duration; + loop { + if let Some(code) = child.try_wait() { return Ok(code); } + if Instant::now() >= deadline { child.kill(); child.wait(); return Ok(124); } + std::thread::sleep(Duration::from_millis(10).min(deadline.saturating_duration_since(Instant::now()))); + } +} +fn main() { std::process::exit(match run() { Ok(code) => code, Err(e) => { eprintln!("timeout: {e}"); 125 } }); } diff --git a/userland/fd/.cargo-ok b/userland/fd/.cargo-ok new file mode 100644 index 0000000..5f8b795 --- /dev/null +++ b/userland/fd/.cargo-ok @@ -0,0 +1 @@ +{"v":1} \ No newline at end of file diff --git a/userland/fd/.cargo/config.toml b/userland/fd/.cargo/config.toml new file mode 100644 index 0000000..f9ba0de --- /dev/null +++ b/userland/fd/.cargo/config.toml @@ -0,0 +1,9 @@ +# On Windows MSVC, statically link the C runtime so that the resulting EXE does +# not depend on the vcruntime DLL. +# +# See: https://github.com/sharkdp/fd/issues/1874 + +[target.x86_64-pc-windows-msvc] +rustflags = ["-C", "target-feature=+crt-static"] +[target.i686-pc-windows-msvc] +rustflags = ["-C", "target-feature=+crt-static"] \ No newline at end of file diff --git a/userland/fd/.cargo_vcs_info.json b/userland/fd/.cargo_vcs_info.json new file mode 100644 index 0000000..fed11ef --- /dev/null +++ b/userland/fd/.cargo_vcs_info.json @@ -0,0 +1,6 @@ +{ + "git": { + "sha1": "193d0df1d0ad455e44506953a2cd80428203ff35" + }, + "path_in_vcs": "" +} \ No newline at end of file diff --git a/userland/fd/.github/FUNDING.yml b/userland/fd/.github/FUNDING.yml new file mode 100644 index 0000000..c81bc07 --- /dev/null +++ b/userland/fd/.github/FUNDING.yml @@ -0,0 +1 @@ +github: [sharkdp, tavianator] diff --git a/userland/fd/.github/ISSUE_TEMPLATE/bug_report.yaml b/userland/fd/.github/ISSUE_TEMPLATE/bug_report.yaml new file mode 100644 index 0000000..d6c8d3c --- /dev/null +++ b/userland/fd/.github/ISSUE_TEMPLATE/bug_report.yaml @@ -0,0 +1,42 @@ +name: Bug Report +description: Report a bug. +title: "[BUG] " +labels: bug +body: + - type: markdown + attributes: + value: | + Please check out the [troubleshooting section](https://github.com/sharkdp/fd#troubleshooting) first. + - type: checkboxes + attributes: + label: Checks + options: + - label: I have read the troubleshooting section and still think this is a bug. + required: true + - type: textarea + id: bug + attributes: + label: "Describe the bug you encountered:" + validations: + required: true + - type: textarea + id: expected + attributes: + label: "Describe what you expected to happen:" + - type: input + id: version + attributes: + label: "What version of `fd` are you using?" + placeholder: "paste the output of `fd --version` here" + validations: + required: true + - type: textarea + id: os + attributes: + label: Which operating system / distribution are you on? + placeholder: | + Unix: paste the output of `uname -srm` and `lsb_release -a` here. + Windows: please tell us your Windows version + render: shell + validations: + required: true diff --git a/userland/fd/.github/ISSUE_TEMPLATE/config.yml b/userland/fd/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 0000000..0086358 --- /dev/null +++ b/userland/fd/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1 @@ +blank_issues_enabled: true diff --git a/userland/fd/.github/ISSUE_TEMPLATE/feature_request.md b/userland/fd/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 0000000..93af6cd --- /dev/null +++ b/userland/fd/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,8 @@ +--- +name: Feature Request +about: Suggest an idea for this project. +title: '' +labels: feature-request +assignees: '' + +--- diff --git a/userland/fd/.github/ISSUE_TEMPLATE/question.md b/userland/fd/.github/ISSUE_TEMPLATE/question.md new file mode 100644 index 0000000..d3e8752 --- /dev/null +++ b/userland/fd/.github/ISSUE_TEMPLATE/question.md @@ -0,0 +1,13 @@ +--- +name: Question +about: Ask a question about 'fd'. +title: '' +labels: question +assignees: '' + +--- + + + +**What version of `fd` are you using?** +[paste the output of `fd --version` here] diff --git a/userland/fd/.github/dependabot.yml b/userland/fd/.github/dependabot.yml new file mode 100644 index 0000000..b69df26 --- /dev/null +++ b/userland/fd/.github/dependabot.yml @@ -0,0 +1,14 @@ +version: 2 +updates: + - package-ecosystem: "cargo" + directory: "/" + schedule: + interval: "monthly" + cooldown: + default-days: 7 + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "daily" + cooldown: + default-days: 7 diff --git a/userland/fd/.github/workflows/CICD.yml b/userland/fd/.github/workflows/CICD.yml new file mode 100644 index 0000000..e3cd2f8 --- /dev/null +++ b/userland/fd/.github/workflows/CICD.yml @@ -0,0 +1,299 @@ +name: CICD + +env: + CICD_INTERMEDIATES_DIR: "_cicd-intermediates" + MSRV_FEATURES: "--all-features" + +on: + workflow_dispatch: + pull_request: + push: + branches: + - master + tags: + - '*' + +permissions: + contents: read + +jobs: + crate_metadata: + name: Extract crate metadata + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7.0.1 + with: + persist-credentials: false + - name: Extract crate information + id: crate_metadata + run: | + echo "name=fd" | tee -a $GITHUB_OUTPUT + cargo metadata --no-deps --format-version 1 | jq -r '"version=" + .packages[0].version' | tee -a $GITHUB_OUTPUT + cargo metadata --no-deps --format-version 1 | jq -r '"maintainer=" + .packages[0].authors[0]' | tee -a $GITHUB_OUTPUT + cargo metadata --no-deps --format-version 1 | jq -r '"homepage=" + .packages[0].homepage' | tee -a $GITHUB_OUTPUT + cargo metadata --no-deps --format-version 1 | jq -r '"msrv=" + .packages[0].rust_version' | tee -a $GITHUB_OUTPUT + outputs: + name: ${{ steps.crate_metadata.outputs.name }} + version: ${{ steps.crate_metadata.outputs.version }} + maintainer: ${{ steps.crate_metadata.outputs.maintainer }} + homepage: ${{ steps.crate_metadata.outputs.homepage }} + msrv: ${{ steps.crate_metadata.outputs.msrv }} + + ensure_cargo_fmt: + name: Ensure 'cargo fmt' has been run + runs-on: ubuntu-22.04 + steps: + - uses: dtolnay/rust-toolchain@stable + with: + components: rustfmt + - uses: actions/checkout@v7.0.1 + with: + persist-credentials: false + - run: cargo fmt -- --check + + lint_check: + name: Ensure 'cargo clippy' has no warnings + runs-on: ubuntu-latest + steps: + - uses: dtolnay/rust-toolchain@stable + with: + components: clippy + - uses: actions/checkout@v7.0.1 + with: + persist-credentials: false + - run: cargo clippy --all-targets --all-features -- -Dwarnings + + min_version: + name: Minimum supported rust version + runs-on: ubuntu-22.04 + needs: crate_metadata + steps: + - name: Checkout source code + uses: actions/checkout@v7.0.1 + with: + persist-credentials: false + + - name: Install rust toolchain (v${{ needs.crate_metadata.outputs.msrv }}) + uses: dtolnay/rust-toolchain@master + with: + toolchain: ${{ needs.crate_metadata.outputs.msrv }} + components: clippy + - name: Run clippy (on minimum supported rust version to prevent warnings we can't fix) + run: cargo clippy --locked --all-targets "${MSRV_FEATURES}" + - name: Run tests + run: cargo test --locked "${MSRV_FEATURES}" + + build: + name: '${{ matrix.job.target }} (${{ matrix.job.os }})' + runs-on: ${{ matrix.job.os }} + needs: crate_metadata + permissions: + id-token: write + contents: write + attestations: write + strategy: + fail-fast: false + matrix: + job: + - { target: aarch64-unknown-linux-gnu , os: ubuntu-24.04, use-cross: true } + - { target: aarch64-unknown-linux-musl , os: ubuntu-24.04, use-cross: true } + - { target: arm-unknown-linux-gnueabihf , os: ubuntu-24.04, use-cross: true } + - { target: arm-unknown-linux-musleabihf, os: ubuntu-24.04, use-cross: true } + - { target: i686-pc-windows-msvc , os: windows-2025 } + - { target: i686-unknown-linux-gnu , os: ubuntu-24.04, use-cross: true } + - { target: i686-unknown-linux-musl , os: ubuntu-24.04, use-cross: true } + - { target: x86_64-apple-darwin , os: macos-26-intel } + - { target: aarch64-apple-darwin , os: macos-26 } + - { target: x86_64-pc-windows-gnu , os: windows-2025 } + - { target: x86_64-pc-windows-msvc , os: windows-2025 } + - { target: aarch64-pc-windows-msvc , os: windows-11-arm } + - { target: aarch64-pc-windows-gnullvm , os: windows-11-arm } + - { target: x86_64-unknown-linux-gnu , os: ubuntu-24.04, use-cross: true } + - { target: x86_64-unknown-linux-musl , os: ubuntu-24.04, use-cross: true } + env: + BUILD_CMD: "${{ matrix.job.use-cross && 'cross' || 'cargo' }}" + target: ${{ matrix.job.target }} + name: ${{ needs.crate_metadata.outputs.name }} + steps: + - name: Checkout source code + uses: actions/checkout@v7.0.1 + with: + persist-credentials: false + + - name: Install Windows ARM64 GNU-family linker + if: matrix.job.target == 'aarch64-pc-windows-gnullvm' + id: msys2 + uses: msys2/setup-msys2@ec48f7c5447b3140e2b088413ae3a55687bccb6e # v2.33.0 + with: + msystem: CLANGARM64 + update: true + install: mingw-w64-clang-aarch64-clang + + - name: Expose Windows ARM64 GNU-family linker + if: matrix.job.target == 'aarch64-pc-windows-gnullvm' + shell: bash + env: + MSYS2_LOCATION: ${{ steps.msys2.outputs.msys2-location }} + run: | + bin_dir=$(cygpath -w "${MSYS2_LOCATION}/clangarm64/bin") + echo "${bin_dir}" >> $GITHUB_PATH + echo "CARGO_TARGET_AARCH64_PC_WINDOWS_GNULLVM_LINKER=${bin_dir}\\clang.exe" >> $GITHUB_ENV + # Link libunwind statically so the release does not need MSYS2's libunwind.dll. + echo "CARGO_TARGET_AARCH64_PC_WINDOWS_GNULLVM_RUSTFLAGS=-C target-feature=+crt-static" >> $GITHUB_ENV + + - name: Install prerequisites + shell: bash + run: | + case ${target} in + arm-unknown-linux-*) sudo apt-get -y update ; sudo apt-get -y install gcc-arm-linux-gnueabihf ;; + aarch64-unknown-linux-gnu) sudo apt-get -y update ; sudo apt-get -y install gcc-aarch64-linux-gnu ;; + esac + + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@stable + with: + targets: ${{ matrix.job.target }} + toolchain: "stable" + + - name: Install cross + if: matrix.job.use-cross + env: + cross_version: "v0.2.5" + package_name: "cross-x86_64-unknown-linux-gnu.tar.gz" + GH_TOKEN: "${{ github.token }}" + run: | + dir="$HOME/.local/bin/" + mkdir -p "$dir" + gh release download --repo cross-rs/cross \ + --pattern "${package_name}" -O - "${cross_version}" \ + | tar -C "$dir" -xz + echo "$dir" >> $GITHUB_PATH + echo "Installed cross $cross_version" >> $GITHUB_STEP_SUMMARY + - name: Show version information (Rust, cargo, GCC) + shell: bash + run: | + gcc --version || true + rustup -V + rustup toolchain list + rustup default + cargo -V + rustc -V + + - name: Build + shell: bash + run: $BUILD_CMD build --locked --release --target="${target}" + + - name: Set binary name & path + id: bin + shell: bash + run: | + # Figure out suffix of binary + EXE_suffix="" + case ${target} in + *-pc-windows-*) EXE_suffix=".exe" ;; + esac; + + # Setup paths + BIN_NAME="${name}${EXE_suffix}" + BIN_PATH="target/${target}/release/${BIN_NAME}" + + # Let subsequent steps know where to find the binary + echo "BIN_PATH=${BIN_PATH}" >> $GITHUB_OUTPUT + echo "BIN_NAME=${BIN_NAME}" >> $GITHUB_OUTPUT + + - name: Set testing options + id: test-options + shell: bash + run: | + # test only library unit tests and binary for arm-type targets + unset CARGO_TEST_OPTIONS + case ${target} in + arm-* | aarch64-*) + CARGO_TEST_OPTIONS="--bin=${name}" ;; + esac + echo "CARGO_TEST_OPTIONS=${CARGO_TEST_OPTIONS}" >> $GITHUB_OUTPUT + + - name: Run tests + shell: bash + env: + cargo_test_options: ${{ steps.test-options.outputs.CARGO_TEST_OPTIONS}} + run: $BUILD_CMD test --locked --target="${target}" "${cargo_test_options}" + + - name: Generate completions + id: completions + shell: bash + run: make completions + + - name: Create tarball + id: package + shell: bash + run: bash scripts/create-archive.sh + env: + BIN_PATH: ${{ steps.bin.outputs.BIN_PATH }} + TARGET: ${{ env.target }} + ARCHIVE_NAME: ${{ env.name }}-v${{ needs.crate_metadata.outputs.version }}-${{ env.target }} + PKG_STAGING: "${{ env.CICD_INTERMEDIATES_DIR }}/package" + + - name: Create Debian package + id: debian-package + shell: bash + if: startsWith(matrix.job.os, 'ubuntu') + run: bash scripts/create-deb.sh + env: + TARGET: ${{ matrix.job.target }} + DPKG_VERSION: ${{ needs.crate_metadata.version }} + BIN_PATH: ${{ steps.bin.outputs.BIN_PATH }} + + - name: "Artifact upload: tarball" + id: upload-tarball + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 + with: + name: ${{ steps.package.outputs.PKG_NAME }} + path: ${{ steps.package.outputs.PKG_PATH }} + compression-level: 0 # tarball is already compressed + + - name: "Artifact upload: Debian package" + id: upload-deb + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 + if: steps.debian-package.outputs.DPKG_NAME + with: + name: ${{ steps.debian-package.outputs.DPKG_NAME }} + path: ${{ steps.debian-package.outputs.DPKG_PATH }} + compression-level: 0 # deb files are already compressed + + - name: Check for release + id: is-release + shell: bash + run: | + unset IS_RELEASE ; if [[ $GITHUB_REF =~ ^refs/tags/v[0-9].* ]]; then IS_RELEASE='true' ; fi + echo "IS_RELEASE=${IS_RELEASE}" >> $GITHUB_OUTPUT + + - name: "Attest artifacts" + uses: actions/attest@508db95dd578ae2727ebd6217d5ba78e4fbda05d # v4 + if: steps.is-release.outputs.IS_RELEASE + with: + subject-path: | + ${{ steps.package.outputs.PKG_PATH }} + ${{ steps.debian-package.outputs.DPKG_PATH }} + + - name: Publish archives and packages + uses: softprops/action-gh-release@efb35369e0ad2afab669f228072c1b0d510eae64 # v3.0.3 + if: steps.is-release.outputs.IS_RELEASE + with: + files: | + ${{ steps.package.outputs.PKG_PATH }} + ${{ steps.debian-package.outputs.DPKG_PATH }} + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + winget: + name: Publish to Winget + runs-on: ubuntu-latest + needs: build + if: startsWith(github.ref, 'refs/tags/v') + steps: + - uses: vedantmgoyal9/winget-releaser@b7370d4d152efdce803a26cc6ca9eee58cb2ad54 # v2 + with: + identifier: sharkdp.fd + installers-regex: '-pc-windows-msvc\.zip$' + token: ${{ secrets.WINGET_TOKEN }} diff --git a/userland/fd/.gitignore b/userland/fd/.gitignore new file mode 100644 index 0000000..90c7e46 --- /dev/null +++ b/userland/fd/.gitignore @@ -0,0 +1,4 @@ +target/ +/autocomplete/ +/package/ +**/*.rs.bk diff --git a/userland/fd/CHANGELOG.md b/userland/fd/CHANGELOG.md new file mode 100644 index 0000000..8dbdd91 --- /dev/null +++ b/userland/fd/CHANGELOG.md @@ -0,0 +1,883 @@ +# 10.5.1 + +## Bugfixes +- Fix `--absolute-path` turning verbatim (`\\?\`) search paths into relative paths on Windows: `\\?\UNC\server\share` is now reported as `\\server\share`, and `\\?\Volume{...}` paths are kept intact, see #2207 (@Duo-ee) +- Make `--ignore` and `--no-hidden` after `-u` override only their half of `-u`, see #2192 (@NavpreetSSidhu) +- Fix inline case-insensitive regex flags incorrectly making other `--and` patterns case-sensitive, see #2184 (@mabunemeh) +- Accept `--format` templates that start with `-` when passed as a separate argument, see #2126 (@vulragrag-star) +- Don't incorrectly escape newlines in error messages, see #2104 +- Restore jemalloc as default allocator on supported systems +- Fix the `cargo binstall` release URL, which named the crate (`fd-find`) instead of the binary (`fd`) and fell back to a third-party mirror +- Fix integer overflow when parsing size constraints, rejecting values that exceed `u64::MAX` instead of panicking, see #2196 (@parneetsingh022). +- Various fixes and updates to zsh completion +- Fix --glob patterns when using --full-path on windows. See #2067. + +# 10.5.0 + +## Features +- Add `--ignore-parent` option to override `--no-ignore-parent`, see #1958 (@tmchow) +- Add `--exact` option to match the entire filename exactly (literal, non-substring). + +## Bugfixes +- Sanitize control characters and bidirectional override characters in filenames + when output goes to a terminal, to prevent terminal escape-sequence injection. + Also reject a placeholder as the executable for `--exec-batch`, while still + allowing it for `--exec`. +- Fix broken symlinks being incorrectly filtered out by `--min-depth` when following links (`--follow`), because their depth was not computed; see #1017 (@hexbinoct). +- Handle invalid working directories gracefully when using `--full-path`, see #1900 (@Xavrir). +- Fire the "search pattern contains a path separator" diagnostic for any pattern containing `/`, not just patterns that happen to name an existing directory. Preserves the legacy Windows behaviour that also flags native `\` separators when the pattern resolves to a real directory. See #1873. +- Also fire the "search pattern contains a path separator" diagnostic for `--and` patterns, not only the primary positional pattern. `--and` patterns are matched against the file name just like the primary pattern, so a path separator in them silently returned zero results. See #1873. +- Fix bug where passing "-" as a directory argument didn't actually search that directory, see #849 (@Sean-Kenneth-Doherty). +- Fix panic when `--changed-before`/`--changed-within` is given an out-of-range `@` Unix timestamp; the value is now rejected gracefully, see #2081 (@nikolauspschuetz). + +# 10.4.2 + +## Bugfixes +- Fixed performance regression due to `--ignore-contain`; see #1913 and #1914 + +# 10.4.1 + +This is just a re-release of 10.4.0 due to an issue with the 10.4.0 release. + +# 10.4.0 + +## Features +- Add `--ignore-contain` option to ignore directories containing a named entry (e.g. to ignore [`CACHEDIR.TAG`](https://bford.info/cachedir/)); see #1727 (@fischman). + +## Bugfixes + +- Fix Windows hyperlink generation for paths with spaces. (#1872) + +- `--print0` combined with `--exec` will now print a `\0` between the output of each entry. Note that if there are multiple instances + of `--exec`, the `\0` will be between each _set_ of commands, _not_ between each individual command run. Fixes #1797. + +- Several bugs were fixed by an update to the `ignore` library used for handling ignore rules + - #1506 + - #1667 + - #1813 + +- Fix completions for alias `fdfind` in deb release, see #1888 (@skane-lukas) + +## Changes + +- Minimum required rust version has been increased to 1.90.0. Notably, this means dropping fully support for intel Mac and Windows 7. +- Statically link the CRT for MSVC builds via Cargo config to avoid runtime DLL dependencies, see #1874 (@FidelSch) + +# 10.3.0 + +## Features + +- Add a hidden `--mindepth` alias for `--min-depth`. (#1617) + + +## Bugfixes + + +## Changes + +- Replace `humantime` crate and `chrono` crate with `jiff` crate, see #1690 (@sorairolake). This has some small changes to the + way dates given to options such `--changed-within` and `--changed-before` including: + - 'M' no longer means "month", as that could be confusing with minutes. Use "mo", "mos", "month" or "months" instead. + - month and year now account for variability in the calander rather than being a hard-coded number of seconds. That is probably + what you would expect, but it is a slight change in behavior. +- aarch64 Windows was added to CI and release artifacts +- Many dependencies were updated +- Better support building on Illumos (there is no automated testing, but some known issues were fixed) + +## Other + +This will be the last release that has been tested on x86_64 Mac OS, since GitHub is +dropping support for runners with that hardware. + +It may also be the last release to use a version of Rust with tier-1 support for +x86_64/intel Macs and Windows 7. + + +# 10.2.0 + +## Features + +- Add --hyperlink option to add OSC 8 hyperlinks to output + + +## Bugfixes + + +## Changes + +- Build windows releases with rust 1.77 so windows 7 is still supported +- Deb packages now include symlink for fdfind to be more consistent with official packages + + +## Other + +# 10.1.0 + +## Features + +- Allow passing an optional argument to `--strip-cwd-prefix` of "always", "never", or "auto". to force whether the cwd prefix is stripped or not. +- Add a `--format` option which allows using a format template for direct ouput similar to the template used for `--exec`. (#1043) + +## Bugfixes +- Fix aarch64 page size again. This time it should actually work. (#1085, #1549) (@tavianator) + + +## Other + +- aarch64-apple-darwin target added to builds on the release page. Note that this is a tier 2 rust target. + +# v10.0.0 + +## Features + +- Add `dir` as an alias to `directory` when using `-t` \ `--type`, see #1460 and #1464 (@Ato2207). +- Add support for @%s date format in time filters similar to GNU date (seconds since Unix epoch for --older/--newer), see #1493 (@nabellows) +- Breaking: No longer automatically ignore `.git` when using `--hidden` with vcs ignore enabled. This reverts the change in v9.0.0. While this feature + was often useful, it also broke some existing workflows, and there wasn't a good way to opt out of it. And there isn't really a good way for us to add + a way to opt out of it. And you can easily get similar behavior by adding `.git/` to your global fdignore file. + See #1457. + +## Bugfixes + +- Respect NO_COLOR environment variable with `--list-details` option. (#1455) +- Fix bug that would cause hidden files to be included despite gitignore rules + if search path is "." (#1461, BurntSushi/ripgrep#2711). +- aarch64 builds now use 64k page sizes with jemalloc. This fixes issues on some systems, such as ARM Macs that + have a larger system page size than the system that the binary was built on. (#1547) +- Address [CVE-2024-24576](https://blog.rust-lang.org/2024/04/09/cve-2024-24576.html), by increasing minimum rust version. + + +## Changes +- Minimum supported rust version is now 1.77.2 + + +# v9.0.0 + +## Performance + +- Performance has been *significantly improved*, both due to optimizations in the underlying `ignore` + crate (#1429), and in `fd` itself (#1422, #1408, #1362) - @tavianator. + [Benchmarks results](https://gist.github.com/tavianator/32edbe052f33ef60570cf5456b59de81) show gains + of 6-8x for full traversals of smaller directories (100k files) and up to 13x for larger directories (1M files). + +- The default number of threads is now constrained to be at most 64. This should improve startup time on + systems with many CPU cores. (#1203, #1410, #1412, #1431) - @tmccombs and @tavianator + +- New flushing behavior when writing output to stdout, providing better performance for TTY and non-TTY + use cases, see #1452 and #1313 (@tavianator). + +## Features + +- Support character and block device file types, see #1213 and #1336 (@cgzones) +- Breaking: `.git/` is now ignored by default when using `--hidden` / `-H`, use `--no-ignore` / `-I` or + `--no-ignore-vcs` to override, see #1387 and #1396 (@skoriop) + +## Bugfixes + +- Fix `NO_COLOR` support, see #1421 (@acuteenvy) + +## Other + +- Fixed documentation typos, see #1409 (@marcospb19) + +## Thanks + +Special thanks to @tavianator for his incredible work on performance in the `ignore` crate and `fd` itself. + + + +# v8.7.1 + +## Bugfixes + +- `-1` properly conflicts with the exec family of options. +- `--max-results` overrides `-1` +- `--quiet` properly conflicts with the exec family of options. This used to be the case, but broke during the switch to clap-derive +- `--changed-within` now accepts a space as well as a "T" as the separator between date and time (due to update of chrono dependency) + +## Other +- Many dependencies were updated +- Some documentation was updated and fixed + +# v8.7.0 + +## Features + +- Add flag --no-require-git to always respect gitignore files, see #1216 (@vegerot) + +## Bugfixes + +- Fix logic for when to use global ignore file. There was a bug where the only case where the + global ignore file wasn't processed was if `--no-ignore` was passed, but neither `--unrestricted` + nor `--no-global-ignore-file` is passed. See #1209 + +# v8.6.0 + +## Features + +- New `--and ` option to add additional patterns that must also be matched. See #315 + and #1139 (@Uthar) +- Added `--changed-after` as alias for `--changed-within`, to have a name consistent with `--changed-before`. + + +## Changes + +- Breaking: On Unix-like systems, `--type executable` now additionally checks if + the file is executable by the current user, see #1106 and #1169 (@ptipiak) + + +## Bugfixes + +- Use fd instead of fd.exe for Powershell completions (when completions are generated on windows) + + +## Other + + +# v8.5.3 + +## Bugfixes + +- Fix completion generation to not include full path of fd command +- Fix build error if completions feature is disabled + +# v8.5.2 + +## Bugfixes + +- Fix --owner option value parsing, see #1163 and #1164 (@tmccombs) + + +# v8.5.1 + +## Bugfixes + +- Fix --threads/-j option value parsing, see #1160 and #1162 (@sharkdp) + + +# v8.5.0 + +## Features + +- `--type executable`/`-t` now works on Windows, see #1051 and #1061 (@tavianator) + +## Bugfixes + +- Fixed differences between piped / non-piped output. This changes `fd`s behavior back to what we + had before 8.3.0, i.e. there will be no leading `./` prefixes, unless `--exec`/`-x`, + `--exec-batch`/`-X`, or `--print0`/`-0` are used. `--strip-cwd-prefix` can be used to strip that + prefix in those cases. See #1046, #1115, and #1121 (@tavianator) +- `fd` could previously crash with a panic due to a race condition in Rusts standard library + (see https://github.com/rust-lang/rust/issues/39364). This has been fixed by switching to a different + message passing implementation, see #1060 and #1146 (@tavianator) +- `fd`s memory usage will not grow unboundedly on huge directory trees, see #1146 (@tavianator) +- fd returns an error when current working directory does not exist while a search path is + specified, see #1072 (@vijfhoek) +- Improved "command not found" error message, see #1083 and #1109 (@themkat) +- Preserve command exit codes when using `--exec-batch`, see #1136 and #1137 (@amesgen) + +## Changes + +- No leading `./` prefix for non-interactive results, see above. +- fd now colorizes paths in parallel, significantly improving performance, see #1148 (@tavianator) +- fd can now avoid `stat` syscalls even when colorizing paths, as long as the color scheme doesn't + require metadata, see #1148 (@tavianator) +- The statically linked `musl` versions of `fd` now use `jmalloc`, leading to a significant performance + improvement, see #1062 (@tavianator) + +## Other + +- Added link back to GitHub in man page and `--help` text, see #1086 (@scottchiefbaker) +- Major update in how `fd` handles command line options internally, see #1067 (@tmccombs) + +# v8.4.0 + +## Features + +- Support multiple `--exec ` instances, see #406 and #960 (@tmccombs) + +## Bugfixes + +- "Argument list too long" errors can not appear anymore when using `--exec-batch`/`-X`, as the command invocations are automatically batched at the maximum possible size, even if `--batch-size` is not given. See #410 and #1020 (@tavianator) + +## Changes + +- Directories are now printed with an additional path separator at the end: `foo/bar/`, see #436 and #812 (@yyogo) +- The `-u` flag was changed to be equivalent to `-HI` (previously, a single `-u` was only equivalent to `-I`). Additional `-u` flags are still allowed, but ignored. See #840 and #986 (@jacksontheel) + +## Other + +- Added installation instructions for RHEL8, see #989 (@ethsol) + + +# v8.3.2 + +## Bugfixes + +- Invalid absolute path on windows when searching from the drive root, see #931 and #936 (@gbarta) + + +# v8.3.1 + +## Bugfixes + +- Stop implying `--no-ignore-parent` when `--no-vcs-ignore` is supplied, see #907, #901, #908 (@tmccombs) +- fd no longer waits for the whole traversal if the only matches arrive within max_buffer_time, see #868 and #895 (@tavianator) +- `--max-results=1` now immediately quits after the first result, see #867 +- `fd -h` does not panic anymore when stdout is closed, see #897 + +## Changes + +- Disable jemalloc on FreeBSD, see #896 (@xanderio) +- Updated man page, see #912 (@rlue) +- Updated zsh completions, see #932 (@tmccombs) + + +# v8.3.0 + +## Performance improvements + +- Colorized output is now significantly faster, see #720 and #853 (@tavianator) +- Writing to stdout is now buffered if the output does not go to a TTY. This increases performance + when the output of `fd` is piped to another program or to a file, see #885 (@tmccombs, original + implementation by @sourlemon207) +- File metadata is now cached between the different filters that require it (e.g. `--owner`, + `--size`), reducing the number of `stat` syscalls when multiple filters are used; see #863 + (@tavianator, original implementation by @alexmaco) + +## Features + +- Don't buffer command output from `--exec` when using a single thread. See #522 +- Add new `-q, --quiet` flag, see #303 (@Asha20) +- Add new `--no-ignore-parent` flag, see #787 (@will459) +- Add new `--batch-size` flag, see #410 (@devonhollowood) +- Add opposing command-line options, see #595 (@Asha20) +- Add support for more filesystem indicators in `LS_COLORS`, see + https://github.com/sharkdp/lscolors/pull/35 (@tavianator) + +## Bugfixes + +- Always show the `./` prefix for search results unless the output is a TTY or `--strip-cwd-prefix` is set, see #760 and #861 (@jcaplan) +- Set default path separator to `/` in MSYS, see #537 and #730 (@aswild) +- fd cannot search files under a RAM disk, see #752 +- fd doesn't show substituted drive on Windows, see #365 +- Properly handle write errors to devices that are full, see #737 +- Use local time zone for time functions (`--change-newer-than`, `--change-older-than`), see #631 (@jacobmischka) +- Support `--list-details` on more platforms (like BusyBox), see #783 +- The filters `--owner`, `--size`, and `--changed-{within,before}` now apply to symbolic links + themselves, rather than the link target, except when `--follow` is specified; see #863 +- Change time comparisons to be exclusive, see #794 (@jacobmischka) + +## Changes + +- Apply custom `--path-separator` to commands run with `--exec(-batch)` and `--list-details`, see #697 (@aswild) + +## Other + +- Many documentation updates + + +# v8.2.1 + +No functional changes with respect to v8.2.0. Bugfix in the release process. + +# v8.2.0 + +## Features + +- Add new `--prune` flag, see #535 (@reima) +- Improved the usability of the time-based options, see #624 and #645 (@gorogoroumaru) +- Add support for exact file sizes in the `--size` filter, see #669 and #696 (@Rogach) +- `fd` now prints an error message if the search pattern requires a leading dot but + `--hidden` is not enabled (Unix only), see #615 + +## Bugfixes + +- Avoid panic when performing limited searches in directories with restricted permissions, see #678 +- Invalid numeric command-line arguments are silently ignored, see #675 +- Disable jemalloc on Android, see #662 +- The `--help` text will be colorless if `NO_COLOR` has been set, see #600 (@xanonid) + +## Changes + +- If `LS_COLORS` is not set (e.g. on Windows), we now provide a more comprehensive default which + includes much more filetypes, see #604 and #682 (mjsir911). + +## Other + +- Added `zsh` completion files, see #654 and #189 (@smancill) + +# v8.1.1 + +## Bugfixes + +- Support colored output on older Windows versions if either (1) `--color=always` is set or (2) the `TERM` environment variable is set. See #469 + +# v8.1.0 + +## Features + +- Add new `--owner [user][:group]` filter. See #307 (pull #581) (@alexmaco) +- Add support for a global ignore file (`~/.config/fd/ignore` on Unix), see #575 (@soedirgo) +- Do not exit immediately if one of the search paths is missing, see #587 (@DJRHails) + +## Bugfixes + +- Reverted a change from fd 8.0 that enabled colors on all Windows terminals (see below) in order to support older Windows versions again, see #577. Unfortunately, this re-opens #469 +- Fix segfault caused by jemalloc on macOS Catalina, see #498 +- Fix `--glob` behavior with empty pattern, see #579 (@SeamusConnor) +- Fix `--list-details` on FreeBSD, DragonFly BSD, OpenBSD and NetBSD. See #573 (@t6) + +## Changes + +- Updated documentation for `--size`, see #584 + +# v8.0.0 + +## Features + +- Add a new `-l`/`--list-details` option to show more details about the search results. This is + basically an alias for `--exec-batch ls -l` with some additional `ls` options. + This can be used in order to: + * see metadata like permissions, owner, file size, modification times (#491) + * see symlink targets (#482) + * achieve a deterministic output order (#324, #196, #159) +- Add a new `--max-results=` option to limit the number of search results, see #472, #476 and #555 + This can be useful to speed up searches in cases where you know that there are only N results. + Using this option is also (slightly) faster than piping to `head -n ` where `fd` can only + exit when it finds the search results ` + 1`. +- Add the alias `-1` for `--max-results=1`, see #561. (@SimplyDanny). +- Add new `--type socket` and `--type pipe` filters, see #511. +- Add new `--min-depth ` and `--exact-depth ` options in addition to the existing option + to limit the maximum depth. See #404. +- Support additional ANSI font styles in `LS_COLORS`: faint, slow blink, rapid blink, dimmed, hidden and strikethrough. + +## Bugfixes + +- Preserve non-UTF8 filenames: invalid UTF-8 filenames are now properly passed to child-processes + when using `--exec`, `--exec-batch` or `--list-details`. In `fd`'s output, we replace non-UTF-8 + sequences with the "�" character. However, if the output of `fd` goes to another process, we + print the actual bytes of the filename. For more details, see #558 and #295. +- `LS_COLORS` entries with unsupported font styles are not completely ignored, see #552 + +## Changes + +- Colored output will now be enabled by default on older Windows versions. + This allows the use of colored output if the terminal supports it (e.g. + MinTTY, Git Bash). On the other hand, this will be a regression for users + on older Windows versions with terminals that do not support ANSI escape + sequences. Affected users can use an alias `fd="fd --color=never"` to + continue using `fd` without colors. There is no change of behavior for + Windows 10. See #469. +- When using `--glob` in combination with `--full-path`, a `*` character does not match a path + separation character (`/` or `\\`) anymore. You can use `**` for that. This allows things like + `fd -p -g '/some/base/path/*/*/*.txt'` which would previously match to arbitrary depths (instead + of exactly two folders below `/some/base/path`. See #404. +- "Legacy" support to use `fd -exec` (with a single dash) has been removed. Use `fd -x` or + `fd --exec` instead. +- Overall improved error handling and error messages. + + +## Other + +- Korean translation of the README, see: [한국어](https://github.com/spearkkk/fd-kor) (@spearkkk) + + +# v7.5.0 + +## Features + +- Added `--one-file-system` (aliases: `--mount`, `--xdev`) to not cross file system boundaries on Unix and Windows, see #507 (@FallenWarrior2k). +- Added `--base-directory` to change the working directory in which `fd` is run, see #509 and #475 (@hajdamak). +- `fd` will not use colored output if the `NO_COLOR` environment variable is set, see #550 and #551 (@metadave). +- `fd --exec` will return exit code 1 if one of the executed commands fails, see #526 and #531 (@fusillicode and @Giuffre) + +## Bug Fixes + +- Fixed 'command not found' error when using zsh completion, see #487 (@barskern). +- `fd -L` should include broken symlinks, see #357 and #497 (@tommilligan, @neersighted and @sharkdp) +- Display directories even if we don't have permission to enter, see #437 (@sharkdp) + +## Changes + +- A flag can now be passed multiple times without producing an error, see #488 and #496 (@rootbid). +- Search results are sorted when using the `-X` option to match the behaviour of piping to `xargs`, see #441 and #524 (@Marcoleni @crash-g). + + +# v7.4.0 + +## Performance improvements + +- Reduce number of `stat` syscalls, improving the performance for searches where file metadata is + required (`--type`, `--size`, `--changed-within`, …), see #434 (@tavianator) +- Use jemalloc by default, improving the performance for almost all searches, see #481. Note that + Windows and `*musl*` builds do not profit from this. + +## Features + +- Added a new `-g`/`--glob` option to switch to glob-based searches (instead of regular expression + based searches). This is accompanied by a new `--regex` option that can be used to switch back, + if users want to `alias fd="fd --glob"`. See #284 +- Added a new `--path-separator ` option which can be useful for Windows users who + want/need `fd` to use `/` instead of `\`, see #428 and #153 (@mookid) +- Added support for hidden files on Windows, see #379 +- When `fd` is run with the `--exec-batch`/`-X` option, it now exposes the exit status of the + command that was run, see #333. +- Exit immediately when Ctrl-C has been pressed twice, see #423 + +## Bugfixes + +- Make `--changed-within`/`--changed-before` work for directories, see #470 + +## Other + +- Pre-built `fd` binaries should now be available for `armhf` targets, see #457 (@detly) +- `fd` is now available on Alpine Linux, see #451 (@5paceToast) +- `fd` is now in the officla FreeBSD repositories, see #412 (@t6) +- Added OpenBSD install instructions, see #421 (@evitalis) +- Added metadata to the Debian package, see #416 (@cathalgarvey) +- `fd` can be installed via npm, see #438 (@pablopunk) + + +# v7.3.0 + +## Features + +- New `--exec-batch `/`-X ` option for batch execution of commands, see #360 (@kimsnj). + This allows you to do things like: + ``` bash + fd … -X vim # open all search results in vim (or any other editor) + fd … -X ls -l # view detailed stats about the search results with 'ls' + fd -e svg -X inkscape # open all SVG files in Inkscape + ``` +- Support for 24-bit color codes (when specified via `LS_COLORS`) as well as + different font styles (bold, italic, underline). + +## Changes + +- A few performance improvements, in particular when printing lots of colorized + results to the console, see #370 +- The `LS_COLORS` handling has been "outsourced" to a separate crate (https://github.com/sharkdp/lscolors) that is now being used by other tools as well: [fselect](https://github.com/jhspetersson/fselect), [lsd](https://github.com/Peltoche/lsd/pull/84). For details, see #363. + +## Other + +- `fd` will be available in Ubuntu Disco DIngo (19.04), see #373 (@sylvestre) +- This release should come with a static ARM binary (`arm-unknown-linux-musleabihf`), see #320 (@duncanfinney) +- Various documentation improvements, see #389 + +## Thanks + +Special thanks to @alexmaco for his awesome work on refactoring and code improvements! (see #401, #398, and #383) + +# v7.2.0 + +## Features + +* Added support for filtering by file modification time by adding two new options `--changed-before ` and `--changed-within <..>`. For more details, see the `--help` text, the man page, the relevant issue #165 and the PR #339 (@kimsnj) +* Added `--show-errors` option to enable the display of filesystem error messages such as "permission denied", see #311 (@psinghal20 and @majecty) +* Added `--maxdepth` as a (hidden) alias for `--max-depth`, see #323 (@mqudsi) +* Added `--search-path` option which can be supplied to replace the positional `path` argument at any position. + +## Changes + +* Loosen strict handling of missing `--ignore-file`, see #280 (@psinghal20) +* Re-enabled `.ignore` files, see #156. + +## Bugfixes + +* `fd` could previously get stuck when run from the root directory in the + presence of zombie processes. This curious bug has been fixed in Rust 1.29 and higher. For more details, see #288, [rust-lang/rust#50619](https://github.com/rust-lang/rust/issues/50619) and [the fix](https://github.com/rust-lang/rust/pull/50630) + +## Other + +* `fd` has officially landed in Debian! See #345 for details. Thanks goes to @sylvestre, @paride and possibly others I don't know about. +* Added Chinese translation of README (@chinanf-boy) + +## Thanks + +A special thanks goes to @joshleeb for his amazing improvements throughout +the code base (new tests, refactoring work and various other things)! + + +# v7.1.0 + +## Features + +* Added `--size` filter option, see #276 (@stevepentland, @JonathanxD and @alexmaco) +* Added `--type empty` (or `-t e`) to search for empty files and/or directories, see #273 + +## Changes + +* With the new version, `.gitignore` files will only be respected in Git repositories, not outside. +* A few performance improvements for `--type` searches, see 641976cf7ad311ba741571ca8b7f02b2654b6955 and 50a2bab5cd52d26d4a3bc786885a2c270ed3b227 + +## Other + +* Starting with this release, we will offer pre-built ARM binaries, see #244 +* Added instructions on how to use `fd` with `emacs`, see #282 (@redguardtoo) +* `fd` is now in the official openSUSE repositories, see #275 (@avindra) +* `fd` is now available via MacPorts, see #291 (@raimue) + + +# v7.0.0 + +## Features + +* Added `--type executable` (or `-t x`) to search for executable files only, see #246 (@PramodBisht) +* Added support for `.fdignore` files, see #156 and #241. +* Added `--ignore-file` option to add custom ignore files, see #156. +* Suggest `--fixed-strings` on invalid regular expressions, see #234 (@PramodBisht) +* Detect when user supplied path instead of pattern, see #235. + +## Changes + +* `.ignore` and `.rgignore` files are not parsed anymore. Use `.fdignore` files + or add custom files via `--ignore-file` instead. +* Updated to `regex-syntax` 0.5 (@cuviper) + +## Bugfixes + +* Properly normalize absolute paths, see #268 +* Invalid utf8 filenames displayed when `-e` is used, see #250 +* If `--type` is used, fifos/sockets/etc. are always shown, see #260 + +## Other + +* Packaging: + * The Arch Linux package is now simply called `fd`. + * There is now a `fd` ebuild for Gentoo Linux. + * There is a `scoop` package for `fd` (Windows). + * There is a `Chocolatey` package for `fd` (Windows). + * There is a Fedora `copr` package for `fd`. + + +# v6.3.0 + +## Features + +* Files with multiple extensions can now be found via `--extension`/`-e`, see #214 (@althonos) + ``` bash + > fd -e tar.gz + ``` + +* Added new `-F`/`--fixed-strings`/`--literal` option that treats the pattern as a literal string instead of a regular expression, see #157 + + ``` bash + > fd -F 'file(1).txt' + ``` + +* Allow `-exec` to work as `--exec`, see #226 (@stevepentland) + +## Bugfixes + +* Fixed `Ctrl-C` handling when using `--exec`, see #224 (@Doxterpepper) + +* Fixed wrong file owner for files in deb package, see #213 + +## Other + +* Replaced old gif by a fancy new SVG screencast (@marionebl) +* Updated [benchmark results](https://github.com/sharkdp/fd#benchmark) (fd has become faster in the meantime!). There is a new repository that hosts several benchmarking scripts for fd: https://github.com/sharkdp/fd-benchmarks + + +# v6.2.0 + +## Features + +* Support for filtering by multiple file extensions and multiple file types, see #199 and #177 + (@tkadur). + + For example, it's possible to search for C++ source or header files: + ``` bash + > fd -e cpp -e c -e cxx -e h pattern + ``` + +## Changes + +* The size of the output buffer (for sorting search results) is now limited to 1000 entries. This + improves the search speed significantly if there are a lot of results, see #191 (@sharkdp). + +## Bugfixes + +* Fix a bug where long-running searches could not be killed via Ctrl-C, see #210 (@Doxterpepper) +* fd's exit codes are now in accordance with Unix standards, see #201 (@Doxterpepper) + +## Other + +* Bash, zsh and fish completion should now work with the Ubuntu `.deb` packages, see #195 and #209 + (@tmccombs and @sharkdp) +* There is a new section on how to set up `fzf` to use `fd` in the + [README](https://github.com/sharkdp/fd#using-fd-with-fzf), see #168. + + +# v6.1.0 + +## Features + +* Support for multiple search paths, see #166 (@Doxterpepper) +* Added `--no-ignore-vcs` option to disable `.gitignore` and other VCS ignore files, + without disabling `.ignore` files - see #156 (@ptzz). + +## Bugfixes + +* Handle terminal signals, see #128 (@Doxterpepper) +* Fixed hang on `--exec` when user input was required, see #178 and #193 (@reima) + +## Other + +* Debian packages are now created via Travis CI and should be available for this and all + future releases (@tmccombs). +* fd is now available on Void Linux (@maxice8) +* The minimum required Rust version is now 1.20 + +## Thanks + +@Doxterpepper deserves a special mention for his great work that is included in this release and +for the support in ticket discussions and concerning Travis CI fixes. Thank you very much! + +Thanks also go out to @tmccombs for the work on Debian packages and for reviewing a lot of pull requests! + +# v6.0.0 + +## Changes + +- The `--exec`/`-x` option does not spawn an intermediate shell anymore. This improves the + performance of parallel command execution and fixes a whole class of (present and potentially + future) problems with shell escaping. The drawback is that shell commands cannot directly be + called with `--exec`. See #155 for the full discussion. These changes have been implemented by + @reima (Thanks!). + +## Bugfixes + +- `--exec` does not escape cmd.exe metacharacters on Windows (see #155, as above). + +## Other + +* *fd* is now available in the FreeBSD ports (@andoriyu) +* The minimal `rustc` version is now checked when building with `cargo`, see #164 (@matematikaadit) +* The output directory for the shell completion files is created if it does not exist (@andoriyu) + + +# v5.0.0 + +## Features + +* Added new `--exec`, `-x` option for parallel command execution (@mmstick, see #84 and #116). See the corresponding [README section](https://github.com/sharkdp/fd#parallel-command-execution) for an introduction. +* Auto-disable color output on unsupported Windows shells like `cmd.exe` (@iology, see #129) +* Added the `--exclude`, `-X` option to suppress certain files/directories in the search results + (see #89). +* Added ripgrep aliases `-u` and `-uu` for `--no-ignore` and `--no-ignore --hidden`, respectively + (@unsignedint, see #92) +* Added `-i`, `--ignore-case` (@iology, see #95) +* Made smart case really smart (@reima, see #103) +* Added RedoxOS support (@goyox86, see #131) + +## Changes + +* The dot `.` can now match newlines in file names (@iology, see #111) +* The short `--type` argument for symlinks has been changed from `s` to `l` (@jcpetkovich, see #83) + +## Bugfixes + +* Various improvements in root-path and symlink handling (@iology, see #82, #107, and #113) +* Fixed absolute path handling on Windows (@reima, #93) +* Fixed: current directory not included when using relative path (see #81) +* Fixed `--type` behavior for unknown file types (@iology, see #150) +* Some fixes around `--exec` (@iology, see #142) + +## Other + +* Major updates and bugfixes to our continuous integration and deployment tooling on Travis + (@matematikaadit, see #149, #145, #133) +* Code style improvements & automatic style checking via `rustfmt` on Travis (@Detegr, see #99) +* Added a man page (@pickfire, see #77) +* *fd* has been relicensed under the dual license MIT/Apache-2.0 (@Detegr, see #105) +* Major refactorings and code improvements (Big thanks to @gsquire, @reima, @iology) +* First version of [`CONTRIBUTING`](https://github.com/sharkdp/fd/blob/master/CONTRIBUTING.md) guidelines +* There is now a Nix package (@mehandes) +* *fd* is now in the official Arch Linux repos (@cassava) +* Improved tooling around shell completion files (@ImbaKnugel, see #124) +* Updated tutorial in the [`README`](https://github.com/sharkdp/fd/blob/master/README.md) +* The minimum required version of Rust has been bumped to 1.19. + +## Thanks + +A *lot* of things have happened since the last release and I'd like to thank all contributors for their great support. I'd also like to thank those that have contributed by reporting bugs and by posting feature requests. + +I'd also like to take this chance to say a special Thank You to a few people that have stood out in one way or another: To @iology, for contributing a multitude of bugfixes, improvements and new features. To @reima and @Detegr for their continuing great support. To @mmstick, for implementing the most advanced new feature of *fd*. And to @matematikaadit for the CI/tooling upgrades. + + +# v4.0.0 + +## Features + +* Added filtering by file extension, for example `fd -e txt`, see #56 (@reima) +* Add option to force colored output: `--color always`, see #49 (@Detegr) +* Generate Shell completions for Bash, ZSH, Fish and Powershell, see #64 (@ImbaKnugel) +* Better & extended `--help` text (@abaez and @Detegr) +* Proper Windows support, see #70 + +## Changes + +* The integration tests have been re-written in Rust :sparkles:, making them platform-independent and easily callable via `cargo test` - see #65 (many thanks to @reima!) +* New tutorial in the README (@deg4uss3r) +* Reduced number of `stat` syscalls for each result from 3 to 1, see #36. +* Enabled Appveyor CI + +# v3.1.0 + +## Features +- Added file type filtering, e.g. `find --type directory` or `find -t f` (@exitium) + +# v3.0.0 + +## Features +- Directories are now traversed in parallel, leading to significant performance improvements (see [benchmarks](https://github.com/sharkdp/fd#benchmark)) +- Added `--print0` option (@michaelmior) +- Added AUR packages (@wezm) + +## Changes +- Changed short flag for `--follow` from `-f` to `-L` (consistency with `ripgrep`) + +# v2.0.0 + +* Changed `--sensitive` to `--case-sensitive` +* Changed `--absolute` to `--absolute-path` +* Throw an error if root directory is not existent, see #39 +* Use absolute paths if the root dir is an absolute path, see #40 +* Handle invalid UTF-8, see #34 #38 +* Support `-V`, `--version` by switching from `getopts` to `clap`. + +Misc: +* It's now possible to install `fd` via homebrew on macOS: `brew install fd`. + +# v1.1.0 + +- Windows compatibility (@sebasv), see #29 #35 +- Safely exit on broken output pipes (e.g.: usage with `head`, `tail`, ..), see #24 +- Backport for rust 1.16, see #23 + +# v1.0.0 + +* Respect `.(git)ignore` files +* Use `LS_COLORS` environment variable directly, instead of `~/.dir_colors` file. +* Added unit and integration tests +* Added optional second argument (search path) + +# v0.3.0 + +- Parse dircolors files, closes #20 +- Colorize each path component, closes #19 +- Add short command line option for --hidden, see #18 + +# v0.2.0 + +- Option to follow symlinks, disable colors, closes #16, closes #17 +- `--filename` instead of `--full-path` +- Option to search hidden directories, closes #12 +- Configurable search depth, closes #13 +- Detect interactive terminal, closes #11 + +# v0.1.0 + +Initial release diff --git a/userland/fd/CONTRIBUTING.md b/userland/fd/CONTRIBUTING.md new file mode 100644 index 0000000..d2e8bc8 --- /dev/null +++ b/userland/fd/CONTRIBUTING.md @@ -0,0 +1,54 @@ +## Contributing to *fd* + +**Thank you very much for considering contributing to this project!** + +We welcome any form of contribution: + + * New issues (feature requests, bug reports, questions, ideas, ...) + * Pull requests (documentation improvements, code improvements, new features, ...) + +**Note**: Before you take the time to open a pull request, please open a ticket first. This will +give us the chance to discuss any potential changes first. + +## Pull Request Expectations + +1. Your contribution should be high quality, and best effort. In particular, please don't make a Pull Request + that was created using generative AI unless you have reviewed it yourself and understand what it does, and be + able to meaningfully answer questions and make changes to it as needed. +2. Your code should successfully compile and pass tests before publishing a Pull Request. It's understandable + if tests fail on one or two specific targets, but if the build doesn't pass for any targets, that is a signal + that the PR is low effort. +3. It is your responsibility to ensure that the generated code does not violate any copyrights or patents. This is + true of all contributions, but is especially relevant when using an LLM that may be generating output that is + very similar or even identical to existing code with an incompatible license. +4. If an AI coding assistant or other LLM tool was used, you must indicate that an AI tool was used, and to what extent it was used. +5. AI should not be used to generate comments when communicating with maintainers. We expect communication with the maintainers, + including PR descriptions and comments, to be from humans, not from an AI. Using an AI to assist with grammar, spelling, and + proofreading or translating is acceptable, as long as the content and voice of the + message is your own. + +## Add an entry to the changelog + +If your contribution changes the behavior of `fd` (as opposed to a typo-fix +in the documentation), please update the [`CHANGELOG.md`](CHANGELOG.md#upcoming-release) file +and describe your changes. This makes the release process much easier and +therefore helps to get your changes into a new `fd` release faster. + +The top of the `CHANGELOG` contains an *"Upcoming release"* section with a few +subsections (Features, Bugfixes, …). Please add your entry to the subsection +that best describes your change. + +Entries follow this format: +``` +- Short description of what has been changed, see #123 (@user) +``` +Here, `#123` is the number of the original issue and/or your pull request. +Please replace `@user` by your GitHub username. + +## Important links + + * [Open issues](https://github.com/sharkdp/fd/issues) + * [Open pull requests](https://github.com/sharkdp/fd/pulls) + * [Development section in the README](https://github.com/sharkdp/fd#development) + * [fd on crates.io](https://crates.io/crates/fd-find) + * [LICENSE-APACHE](https://github.com/sharkdp/fd/blob/master/LICENSE-APACHE) and [LICENSE-MIT](https://github.com/sharkdp/fd/blob/master/LICENSE-MIT) diff --git a/userland/fd/Cargo.lock b/userland/fd/Cargo.lock new file mode 100644 index 0000000..95d1c74 --- /dev/null +++ b/userland/fd/Cargo.lock @@ -0,0 +1,883 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "aho-corasick" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" +dependencies = [ + "memchr", +] + +[[package]] +name = "anstream" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + +[[package]] +name = "anstyle-parse" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys", +] + +[[package]] +name = "anyhow" +version = "1.0.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" + +[[package]] +name = "argmax" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0144c58b55af0133ec3963ce5e4d07aad866e3bbcfdcddbf4590dbd7ad6ff557" +dependencies = [ + "libc", + "nix 0.30.1", + "once_cell", +] + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "bitflags" +version = "2.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" + +[[package]] +name = "block2" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdeb9d870516001442e364c5220d3574d2da8dc765554b4a617230d33fa58ef5" +dependencies = [ + "objc2", +] + +[[package]] +name = "bstr" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6bb31b46c14244e20ee9984b11bf5c992b91fb6939fea616e3512c8baecdbe5f" +dependencies = [ + "memchr", + "serde_core", +] + +[[package]] +name = "cc" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86e055fa6e3e40ad6187ba25b6855eeb635d5271f8e0f0e1e9a5826b8d75bea0" +dependencies = [ + "find-msvc-tools", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600" + +[[package]] +name = "cfg_aliases" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" + +[[package]] +name = "clap" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa8876b300ab35ba921adea3dfd70157a46249b33f95c9084ae5709785478946" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0797fb7aeb1406c84efac526901f7ec3ead2124f946b494e72879d4b54704d" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", + "terminal_size", +] + +[[package]] +name = "clap_complete" +version = "4.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "037e2a1a92236d0aff7e845093f64661d6df4c02c9fcc61a60e9e1d736fa392f" +dependencies = [ + "clap", +] + +[[package]] +name = "clap_derive" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9c751b79415d4e559e3d1fcf128e09e720eb673a06d26cf6f392d37d75b66e0" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "clap_lex" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c133bc6a41be0d194c306b5506d15e6feeea7b1d6604bd3f8310dfb2ca96486" + +[[package]] +name = "colorchoice" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" + +[[package]] +name = "crossbeam-channel" +version = "0.5.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "98b0cc327b5bc766e7fda9c9260cc0fa81b43a8e240440422dff70788e3f9ef1" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "622f3fc73690be383c7214310406f28a90e6edeadc3cea882f9d71e495b9711a" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc74980687109a3b14c72fd458107bf0baa1da1a1a805e178d15501ba9b86d9d" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a31eee39dddec8330830986fcd7625edb5a24ec90ea038215273bbc3adb08ac6" + +[[package]] +name = "ctrlc" +version = "3.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0b1fab2ae45819af2d0731d60f2afe17227ebb1a1538a236da84c93e9a60162" +dependencies = [ + "dispatch2", + "nix 0.31.3", + "windows-sys", +] + +[[package]] +name = "defmt" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4a0795ac740cb54d7b3775c9c17345f6aaecf884c01733b39d6017738e0e3e1" +dependencies = [ + "bitflags 1.3.2", + "defmt-macros", +] + +[[package]] +name = "defmt-macros" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "840bb056bd99c495c9188198a258ab787c79f306d864db3e6e8451442d6d8366" +dependencies = [ + "defmt-parser", + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "defmt-parser" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71958f69fb2de856c5dd51ac9377e986ca1aa5919f931bf5395a807277f889a0" +dependencies = [ + "thiserror", +] + +[[package]] +name = "diff" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56254986775e3233ffa9c4d7d3faaf6d36a2c09d30b20687e9f88bc8bafc16c8" + +[[package]] +name = "dispatch2" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e0e367e4e7da84520dedcac1901e4da967309406d1e51017ae1abfb97adbd38" +dependencies = [ + "bitflags 2.13.2", + "block2", + "libc", + "objc2", +] + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "etcetera" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de48cc4d1c1d97a20fd819def54b890cadde72ed3ad0c614822a0a433361be96" +dependencies = [ + "cfg-if", + "windows-sys", +] + +[[package]] +name = "faccess" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ae66425802d6a903e268ae1a08b8c38ba143520f227a205edf4e9c7e3e26d5" +dependencies = [ + "bitflags 1.3.2", + "libc", + "winapi", +] + +[[package]] +name = "fastrand" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" + +[[package]] +name = "fd-find" +version = "10.5.1" +dependencies = [ + "aho-corasick", + "anyhow", + "argmax", + "clap", + "clap_complete", + "crossbeam-channel", + "ctrlc", + "diff", + "etcetera", + "faccess", + "filetime", + "globset", + "ignore", + "jiff", + "libc", + "lscolors", + "memchr", + "nix 0.31.3", + "normpath", + "nu-ansi-term", + "regex", + "regex-syntax", + "tempfile", + "test-case", + "tikv-jemallocator", +] + +[[package]] +name = "filetime" +version = "0.2.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c287a33c7f0a620c38e641e7f60827713987b3c0f26e8ddc9462cc69cf75759" +dependencies = [ + "cfg-if", + "libc", +] + +[[package]] +name = "find-msvc-tools" +version = "0.1.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c66915cc5391097de7a2b0b972e8505aa33c0df401fd3a6811dbdb32881ba48" + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "libc", + "r-efi", +] + +[[package]] +name = "globset" +version = "0.4.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f98b0d85522dc0a824cfebbc9594ff4a9b289c568bea0370319dfbdf20ccdd2d" +dependencies = [ + "aho-corasick", + "bstr", + "log", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "ignore" +version = "0.4.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "484be5657551d7868d67587df967fb4fcbe9d266a761cc855aaa148fda0b3cc7" +dependencies = [ + "crossbeam-deque", + "globset", + "log", + "memchr", + "regex-automata", + "same-file", + "walkdir", + "winapi-util", +] + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + +[[package]] +name = "jiff" +version = "0.2.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2b005715dcbeb0089a3c0dab99f2ff1cc3b2525323552703d648585d342a383" +dependencies = [ + "defmt", + "jiff-core", + "jiff-static", + "jiff-tzdb-platform", + "log", + "portable-atomic", + "portable-atomic-util", + "serde_core", + "windows-link", +] + +[[package]] +name = "jiff-core" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e52fe76043ccecc9005d2305ebaadf7d7fc0cc89ca6baa10a94d6bc68c7128c" +dependencies = [ + "defmt", + "log", +] + +[[package]] +name = "jiff-static" +version = "0.2.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2cc9817253cf7c7ee4684451bd327e88d6f3658014e54a29198625590650695c" +dependencies = [ + "jiff-core", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "jiff-tzdb" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa8377070c6bae868759445e5a77f66d84f0b72f3a054bfb00e6d038b8282da7" + +[[package]] +name = "jiff-tzdb-platform" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "875a5a69ac2bab1a891711cf5eccbec1ce0341ea805560dcd90b7a2e925132e8" +dependencies = [ + "jiff-tzdb", +] + +[[package]] +name = "libc" +version = "0.2.191" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65cd87c9a6d60536580b316ad985726104730a5151c99845f2490428ef91240a" + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "log" +version = "0.4.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" + +[[package]] +name = "lscolors" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d60e266dfb1426eb2d24792602e041131fdc0236bb7007abc0e589acafd60929" +dependencies = [ + "aho-corasick", + "nu-ansi-term", +] + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "nix" +version = "0.30.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74523f3a35e05aba87a1d978330aef40f67b0304ac79c1c00b294c9830543db6" +dependencies = [ + "bitflags 2.13.2", + "cfg-if", + "cfg_aliases", + "libc", +] + +[[package]] +name = "nix" +version = "0.31.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf20d2fde8ff38632c426f1165ed7436270b44f199fc55284c38276f9db47c3d" +dependencies = [ + "bitflags 2.13.2", + "cfg-if", + "cfg_aliases", + "libc", +] + +[[package]] +name = "normpath" +version = "1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b11ce00d2594068e8a27c9146fdc5cf9f3ac38eb42c7cd34d05fea618873ac9f" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "nu-ansi-term" +version = "0.50.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "objc2" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08849bbd4767dfae9457696856ae1c84fe4e0281bbe4a7abff2d0e06fb7981f8" +dependencies = [ + "objc2-encode", +] + +[[package]] +name = "objc2-encode" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef25abbcd74fb2609453eb695bd2f860d389e457f67dc17cafc8b8cbc89d0c33" + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + +[[package]] +name = "portable-atomic" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85" + +[[package]] +name = "portable-atomic-util" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10ab3eb7f3becc3a1cbc4f2c6f20267996cfc1a6467a873763411b136a122715" +dependencies = [ + "portable-atomic", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "regex" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "rustix" +version = "1.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5e76742e5294153d4dde1207dbec5bdd81eef6fabdc0d2118babadabfd858aa" +dependencies = [ + "bitflags 2.13.2", + "errno", + "libc", + "linux-raw-sys", + "windows-sys", +] + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d78c8dee4c7bf0e14673097256fed6142ce9d3b85a408189d07482442145823b" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom", + "once_cell", + "rustix", + "windows-sys", +] + +[[package]] +name = "terminal_size" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "230a1b821ccbd75b185820a1f1ff7b14d21da1e442e22c0863ea5f08771a8874" +dependencies = [ + "rustix", + "windows-sys", +] + +[[package]] +name = "test-case" +version = "3.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "124953e7f67cb0b2fcfb87e899e4ae5a64fb68e2e160767933cc67b646fb0042" +dependencies = [ + "test-case-macros", +] + +[[package]] +name = "test-case-core" +version = "3.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd097615b407247e102046ba34426bb594923ab3554cf0f3cdfed050f1a5a3e8" +dependencies = [ + "cfg-if", + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "test-case-macros" +version = "3.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4ce35ece947bccac166e1ded639133827a953f69a48ba134d17f91108333082" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", + "test-case-core", +] + +[[package]] +name = "thiserror" +version = "2.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09e52cb86a36cede5cb101bf8908837b3e4c6e5e59fe7fd85c23fb56200d189e" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe5197923287db20a58125f0bc85c062f7f2c892de97b18c356f9efb14b28524" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "tikv-jemalloc-sys" +version = "0.7.1+5.3.1-0-g81034ce1f1373e37dc865038e1bc8eeecf559ce8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a2825c78386b4ae0314074867860ba9577875de945f05992c38815cbec327f0" +dependencies = [ + "cc", + "libc", +] + +[[package]] +name = "tikv-jemallocator" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "249f09e49ab1609436f34c776e84231bead18d6a955f119f939bdc1d847561bd" +dependencies = [ + "libc", + "tikv-jemalloc-sys", +] + +[[package]] +name = "unicode-ident" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2c754d6c33795a1c324727428e5a7dedb5b06195f9890bdbcba760d3e246563" + +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] diff --git a/userland/fd/Cargo.toml b/userland/fd/Cargo.toml new file mode 100644 index 0000000..080e6ba --- /dev/null +++ b/userland/fd/Cargo.toml @@ -0,0 +1,183 @@ +# THIS FILE IS AUTOMATICALLY GENERATED BY CARGO +# +# When uploading crates to the registry Cargo will automatically +# "normalize" Cargo.toml files for maximal compatibility +# with all versions of Cargo and also rewrite `path` dependencies +# to registry (e.g., crates.io) dependencies. +# +# If you are reading this file be aware that the original Cargo.toml +# will likely look very different (and much more reasonable). +# See Cargo.toml.orig for the original contents. + +[package] +edition = "2024" +rust-version = "1.90.0" +name = "fd-find" +version = "10.5.1" +authors = ["David Peter "] +build = false +exclude = ["/benchmarks/*"] +autolib = false +autobins = false +autoexamples = false +autotests = false +autobenches = false +description = "fd is a simple, fast and user-friendly alternative to find." +homepage = "https://github.com/sharkdp/fd" +documentation = "https://docs.rs/fd-find" +readme = "README.md" +keywords = [ + "search", + "find", + "file", + "filesystem", + "tool", +] +categories = ["command-line-utilities"] +license = "MIT OR Apache-2.0" +repository = "https://github.com/sharkdp/fd" + +[package.metadata.binstall] +pkg-url = "{ repo }/releases/download/v{ version }/fd-v{ version }-{ target }.{ archive-format }" +bin-dir = "{ bin }-v{ version }-{ target }/{ bin }{ binary-ext }" +pkg-fmt = "tgz" + +[package.metadata.binstall.overrides.x86_64-pc-windows-msvc] +pkg-fmt = "zip" + +[package.metadata.binstall.overrides.x86_64-pc-windows-gnu] +pkg-fmt = "zip" + +[package.metadata.binstall.overrides.i686-pc-windows-msvc] +pkg-fmt = "zip" + +[package.metadata.binstall.overrides.aarch64-pc-windows-msvc] +pkg-fmt = "zip" + +[badges.appveyor] +repository = "sharkdp/fd" + +[badges.travis-ci] +repository = "sharkdp/fd" + +[features] +base = ["use-jemalloc"] +completions = ["clap_complete"] +# WASI uses the standard allocator. +default = [] +use-jemalloc = ["tikv-jemallocator"] + +[[bin]] +name = "fd" +path = "src/main.rs" + +[[test]] +name = "tests" +path = "tests/tests.rs" + +[dependencies.aho-corasick] +version = "1.1" + +[dependencies.anyhow] +version = "1.0" + +[dependencies.argmax] +version = "0.4.0" + +[dependencies.clap] +version = "4.6.1" +features = [ + "suggestions", + "color", + "wrap_help", + "cargo", + "derive", +] + +[dependencies.clap_complete] +version = "4.6.5" +optional = true + +[dependencies.crossbeam-channel] +version = "0.5.15" + +[target.'cfg(not(target_os = "wasi"))'.dependencies.ctrlc] +version = "3.5" + +[dependencies.etcetera] +version = "0.11" + +[dependencies.faccess] +version = "0.2.4" + +[dependencies.globset] +version = "0.4.19" + +[dependencies.ignore] +version = "0.4.28" + +[dependencies.jiff] +version = "0.2.27" + +[dependencies.lscolors] +version = "0.21" +features = ["nu-ansi-term"] +default-features = false + +[dependencies.memchr] +version = "2.8.0" + +[dependencies.normpath] +version = "1.5.1" + +[dependencies.nu-ansi-term] +version = "0.50" + +[dependencies.regex] +version = "1.12.2" + +[dependencies.regex-syntax] +version = "0.8" + +[dev-dependencies.diff] +version = "0.1" + +[dev-dependencies.filetime] +version = "0.2" + +[dev-dependencies.tempfile] +version = "3.27" + +[dev-dependencies.test-case] +version = "3.3" + +[target.'cfg(all(not(windows), not(target_os = "android"), not(target_os = "macos"), not(target_os = "freebsd"), not(target_os = "openbsd"), not(target_os = "illumos"), not(all(target_env = "musl", target_pointer_width = "32")), not(target_arch = "riscv64")))'.dependencies.tikv-jemallocator] +version = "0.7.0" +optional = true + +[target.'cfg(all(unix, not(target_os = "redox")))'.dependencies.libc] +version = "0.2" + +[target."cfg(unix)".dependencies.nix] +version = "0.31.1" +features = [ + "signal", + "user", + "hostname", +] +default-features = false + +[profile.debugging] +debug = 2 +inherits = "dev" + +[profile.dev] +debug = "line-tables-only" + +[profile.dev.package."*"] +debug = 0 + +[profile.release] +lto = true +codegen-units = 1 +strip = true diff --git a/userland/fd/Cargo.toml.orig b/userland/fd/Cargo.toml.orig new file mode 100644 index 0000000..b995b7b --- /dev/null +++ b/userland/fd/Cargo.toml.orig @@ -0,0 +1,116 @@ +[package] +authors = ["David Peter "] +categories = ["command-line-utilities"] +description = "fd is a simple, fast and user-friendly alternative to find." +exclude = ["/benchmarks/*"] +homepage = "https://github.com/sharkdp/fd" +documentation = "https://docs.rs/fd-find" +keywords = [ + "search", + "find", + "file", + "filesystem", + "tool", +] +license = "MIT OR Apache-2.0" +name = "fd-find" +readme = "README.md" +repository = "https://github.com/sharkdp/fd" +version = "10.5.1" +edition= "2024" +rust-version = "1.90.0" + +[badges.appveyor] +repository = "sharkdp/fd" + +[badges.travis-ci] +repository = "sharkdp/fd" + +[[bin]] +name = "fd" +path = "src/main.rs" + +[dependencies] +aho-corasick = "1.1" +nu-ansi-term = "0.50" +argmax = "0.4.0" +ignore = "0.4.28" +regex = "1.12.2" +regex-syntax = "0.8" +ctrlc = "3.5" +globset = "0.4.19" +anyhow = "1.0" +etcetera = "0.11" +normpath = "1.5.1" +crossbeam-channel = "0.5.15" +clap_complete = {version = "4.6.5", optional = true} +faccess = "0.2.4" +jiff = "0.2.27" +memchr = "2.8.0" + +[dependencies.clap] +version = "4.6.1" +features = ["suggestions", "color", "wrap_help", "cargo", "derive"] + +[dependencies.lscolors] +version = "0.21" +default-features = false +features = ["nu-ansi-term"] + +[target.'cfg(unix)'.dependencies] +nix = { version = "0.31.1", default-features = false, features = ["signal", "user", "hostname"] } + +[target.'cfg(all(unix, not(target_os = "redox")))'.dependencies] +libc = "0.2" + +# FIXME: Re-enable jemalloc on macOS +# jemalloc is currently disabled on macOS due to a bug in jemalloc in combination with macOS +# Catalina. See https://github.com/sharkdp/fd/issues/498 for details. +# This has to be kept in sync with src/main.rs where the allocator for +# the program is set. +[target.'cfg(all(not(windows), not(target_os = "android"), not(target_os = "macos"), not(target_os = "freebsd"), not(target_os = "openbsd"), not(target_os = "illumos"), not(all(target_env = "musl", target_pointer_width = "32")), not(target_arch = "riscv64")))'.dependencies] +tikv-jemallocator = {version = "0.7.0", optional = true} + +[dev-dependencies] +diff = "0.1" +tempfile = "3.27" +filetime = "0.2" +test-case = "3.3" + +[profile.dev] +debug = "line-tables-only" + +[profile.dev.package."*"] +debug = false + +[profile.debugging] +inherits = "dev" +debug = true + +[profile.release] +lto = true +strip = true +codegen-units = 1 + +[features] +use-jemalloc = ["tikv-jemallocator"] +completions = ["clap_complete"] +base = ["use-jemalloc"] +default = ["use-jemalloc", "completions"] + +[package.metadata.binstall] +pkg-url = "{ repo }/releases/download/v{ version }/fd-v{ version }-{ target }.{ archive-format }" +bin-dir = "{ bin }-v{ version }-{ target }/{ bin }{ binary-ext }" +pkg-fmt = "tgz" + +[package.metadata.binstall.overrides.x86_64-pc-windows-msvc] +pkg-fmt = "zip" + +[package.metadata.binstall.overrides.x86_64-pc-windows-gnu] +pkg-fmt = "zip" + +[package.metadata.binstall.overrides.i686-pc-windows-msvc] +pkg-fmt = "zip" + +[package.metadata.binstall.overrides.aarch64-pc-windows-msvc] +pkg-fmt = "zip" diff --git a/userland/fd/Cross.toml b/userland/fd/Cross.toml new file mode 100644 index 0000000..3a14621 --- /dev/null +++ b/userland/fd/Cross.toml @@ -0,0 +1,6 @@ +# https://github.com/sharkdp/fd/issues/1085 +[target.aarch64-unknown-linux-gnu.env] +passthrough = ["JEMALLOC_SYS_WITH_LG_PAGE=16"] + +[target.aarch64-unknown-linux-musl.env] +passthrough = ["JEMALLOC_SYS_WITH_LG_PAGE=16"] diff --git a/userland/fd/LICENSE-APACHE b/userland/fd/LICENSE-APACHE new file mode 100644 index 0000000..c9d7204 --- /dev/null +++ b/userland/fd/LICENSE-APACHE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + +Copyright 2017-2020 fd developers + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/userland/fd/LICENSE-MIT b/userland/fd/LICENSE-MIT new file mode 100644 index 0000000..dfc8683 --- /dev/null +++ b/userland/fd/LICENSE-MIT @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2017-present The fd developers + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/userland/fd/Makefile b/userland/fd/Makefile new file mode 100644 index 0000000..78ab0a9 --- /dev/null +++ b/userland/fd/Makefile @@ -0,0 +1,64 @@ +PROFILE?=release +EXE=target/$(PROFILE)/fd +prefix=/usr/local +bindir=$(prefix)/bin +datadir=$(prefix)/share +exe_name=fd +ifdef VERSION + ARCHIVE_NAME = fd-v$(VERSION) +else + ARCHIVE_NAME = fd +endif +export ARCHIVE_NAME +archive_path=package/$(ARCHIVE_NAME).tar.gz + +$(EXE): Cargo.toml src/**/*.rs + cargo build --profile $(PROFILE) --locked + +.PHONY: completions +completions: autocomplete/fd.bash autocomplete/fd.fish autocomplete/_fd.ps1 autocomplete/_fd autocomplete/_fdfind autocomplete/fdfind.bash autocomplete/fdfind.fish + +comp_dir=@mkdir -p autocomplete + +autocomplete/fd.bash: $(EXE) + $(comp_dir) + $(EXE) --gen-completions bash > $@ + +autocomplete/fd.fish: $(EXE) + $(comp_dir) + $(EXE) --gen-completions fish > $@ + +autocomplete/_fd.ps1: $(EXE) + $(comp_dir) + $(EXE) --gen-completions powershell > $@ + +autocomplete/_fd: contrib/completion/_fd + $(comp_dir) + cp $< $@ + +autocomplete/_fdfind: contrib/completion/_fdfind + $(comp_dir) + cp $< $@ + +autocomplete/fdfind.bash: contrib/completion/fdfind.bash + $(comp_dir) + cp $< $@ + +autocomplete/fdfind.fish: contrib/completion/fdfind.fish + $(comp_dir) + cp $< $@ + +archive: $(archive_path) + +$(archive_path): completions $(EXE) + bash scripts/create-archive.sh + +install: $(EXE) completions + install -Dm755 $(EXE) $(DESTDIR)$(bindir)/fd + install -Dm644 autocomplete/fd.bash $(DESTDIR)/$(datadir)/bash-completion/completions/$(exe_name) + install -Dm644 autocomplete/fd.fish $(DESTDIR)/$(datadir)/fish/vendor_completions.d/$(exe_name).fish + install -Dm644 autocomplete/_fd $(DESTDIR)/$(datadir)/zsh/site-functions/_fd + install -Dm644 autocomplete/fdfind.bash $(DESTDIR)/$(datadir)/zsh/site-functions/fdfind.bash + install -Dm644 autocomplete/fdfind.fish $(DESTDIR)/$(datadir)/zsh/site-functions/fdfind.fish + install -Dm644 autocomplete/_fdfind $(DESTDIR)/$(datadir)/zsh/site-functions/_fdfind + install -Dm644 doc/fd.1 $(DESTDIR)/$(datadir)/man/man1/$(exe_name).1 diff --git a/userland/fd/PATCHES.md b/userland/fd/PATCHES.md new file mode 100644 index 0000000..03d8f01 --- /dev/null +++ b/userland/fd/PATCHES.md @@ -0,0 +1,15 @@ +# WASI patches + +Source: crates.io `fd-find` 10.5.1, https://github.com/sharkdp/fd. +The vendor copy retains the upstream licenses. + +- `Cargo.toml` disables the default allocator and completion features. +- `Cargo.toml` excludes ctrlc on WASI. +- `src/main.rs` selects one thread on WASI and disables the unavailable filesystem-boundary option. +- `src/filesystem.rs` uses the existing false-returning special-file predicates on WASI. +- `src/walk.rs` uses a sequential walker and an unbounded result queue on WASI. +- `src/wasi_walk.rs` adapts the existing visitor to the sequential ignore walker. +- `src/main.rs` and `src/walk.rs` conditionally import native-only and completion-only items. + +The WASI search path consumes results after the sequential walk. +The optional upstream child-execution modes remain unsupported on WASI. diff --git a/userland/fd/README.md b/userland/fd/README.md new file mode 100644 index 0000000..02edbb5 --- /dev/null +++ b/userland/fd/README.md @@ -0,0 +1,807 @@ +# fd + +[![CICD](https://github.com/sharkdp/fd/actions/workflows/CICD.yml/badge.svg)](https://github.com/sharkdp/fd/actions/workflows/CICD.yml) +[![Version info](https://img.shields.io/crates/v/fd-find.svg)](https://crates.io/crates/fd-find) +[[中文](https://github.com/cha0ran/fd-zh)] +[[한국어](https://github.com/spearkkk/fd-kor)] + +`fd` is a program to find entries in your filesystem. +It is a simple, fast and user-friendly alternative to [`find`](https://www.gnu.org/software/findutils/). +While it does not aim to support all of `find`'s powerful functionality, it provides sensible +(opinionated) defaults for a majority of use cases. + +[Installation](#installation) • [How to use](#how-to-use) • [Troubleshooting](#troubleshooting) + +## Features + +* Intuitive syntax: `fd PATTERN` instead of `find -iname '*PATTERN*'`. +* Regular expression (default) and glob-based patterns. +* [Very fast](#benchmark) due to parallelized directory traversal. +* Uses colors to highlight different file types (same as `ls`). +* Supports [parallel command execution](#command-execution) +* Smart case: the search is case-insensitive by default. It switches to + case-sensitive if the pattern contains an uppercase + character[\*](http://vimdoc.sourceforge.net/htmldoc/options.html#'smartcase'). +* Ignores hidden directories and files, by default. +* Ignores patterns from your `.gitignore`, by default. +* The command name is *50%* shorter[\*](https://github.com/ggreer/the_silver_searcher) than + `find` :-). + +## Demo + +![Demo](doc/screencast.svg) + +## How to use + +First, to get an overview of all available command line options, you can either run +[`fd -h`](#command-line-options) for a concise help message or `fd --help` for a more detailed +version. + +### Simple search + +*fd* is designed to find entries in your filesystem. The most basic search you can perform is to +run *fd* with a single argument: the search pattern. For example, assume that you want to find an +old script of yours (the name included `netflix`): +``` bash +> fd netfl +Software/python/imdb-ratings/netflix-details.py +``` +If called with just a single argument like this, *fd* searches the current directory recursively +for any entries that *contain* the pattern `netfl`. + +### Regular expression search + +The search pattern is treated as a regular expression. Here, we search for entries that start +with `x` and end with `rc`: +``` bash +> cd /etc +> fd '^x.*rc$' +X11/xinit/xinitrc +X11/xinit/xserverrc +``` + +The regular expression syntax used by `fd` is [documented here](https://docs.rs/regex/latest/regex/#syntax). + +### Specifying the root directory + +If we want to search a specific directory, it can be given as a second argument to *fd*: +``` bash +> fd passwd /etc +/etc/default/passwd +/etc/pam.d/passwd +/etc/passwd +``` + +### List all files, recursively + +*fd* can be called with no arguments. This is very useful to get a quick overview of all entries +in the current directory, recursively (similar to `ls -R`): +``` bash +> cd fd/tests +> fd +testenv +testenv/mod.rs +tests.rs +``` + +If you want to use this functionality to list all files in a given directory, you have to use +a catch-all pattern such as `.` or `^`: +``` bash +> fd . fd/tests/ +testenv +testenv/mod.rs +tests.rs +``` + +### Searching for a particular file extension + +Often, we are interested in all files of a particular type. This can be done with the `-e` (or +`--extension`) option. Here, we search for all Markdown files in the fd repository: +``` bash +> cd fd +> fd -e md +CONTRIBUTING.md +README.md +``` + +The `-e` option can be used in combination with a search pattern: +``` bash +> fd -e rs mod +src/fshelper/mod.rs +src/lscolors/mod.rs +tests/testenv/mod.rs +``` + +### Searching for a particular file name + + To find files with exactly the provided search pattern, use the `-g` (or `--glob`) option: +``` bash +> fd -g libc.so /usr +/usr/lib32/libc.so +/usr/lib/libc.so +``` + +### Hidden and ignored files +By default, *fd* does not search hidden directories and does not show hidden files in the +search results. To disable this behavior, we can use the `-H` (or `--hidden`) option: +``` bash +> fd pre-commit +> fd -H pre-commit +.git/hooks/pre-commit.sample +``` + +If we work in a directory that is a Git repository (or includes Git repositories), *fd* does not +search folders (and does not show files) that match one of the `.gitignore` patterns. To disable +this behavior, we can use the `-I` (or `--no-ignore`) option: +``` bash +> fd num_cpu +> fd -I num_cpu +target/debug/deps/libnum_cpus-f5ce7ef99006aa05.rlib +``` + +To really search *all* files and directories, simply combine the hidden and ignore features to show +everything (`-HI`) or use `-u`/`--unrestricted`. + +### Matching the full path +By default, *fd* only matches the filename of each file. However, using the `--full-path` or `-p` option, +you can match against the full path. + +```bash +> fd -u -p -g '**/.git/config' +> fd -p '.*/lesson-\d+/[a-z]+.(jpg|png)' +``` + +### Command execution + +Instead of just showing the search results, you often want to *do something* with them. `fd` +provides two ways to execute external commands for each of your search results: + +* The `-x`/`--exec` option runs an external command *for each of the search results* (in parallel). +* The `-X`/`--exec-batch` option launches the external command once, with *all search results as arguments*. + +#### Examples + +Recursively find all zip archives and unpack them: +``` bash +fd -e zip -x unzip +``` +If there are two such files, `file1.zip` and `backup/file2.zip`, this would execute +`unzip file1.zip` and `unzip backup/file2.zip`. The two `unzip` processes run in parallel +(if the files are found fast enough). + +Find all `*.h` and `*.cpp` files and auto-format them inplace with `clang-format -i`: +``` bash +fd -e h -e cpp -x clang-format -i +``` +Note how the `-i` option to `clang-format` can be passed as a separate argument. This is why +we put the `-x` option last. + +Any positional arguments after `-x` belong to the command template, not to `fd` itself. If you +also want to pass a pattern or search path, put `-x` last: +``` bash +fd pattern path -x echo +``` + +Find all `test_*.py` files and open them in your favorite editor: +``` bash +fd -g 'test_*.py' -X vim +``` +Note that we use capital `-X` here to open a single `vim` instance. If there are two such files, +`test_basic.py` and `lib/test_advanced.py`, this will run `vim test_basic.py lib/test_advanced.py`. + +To see details like file permissions, owners, file sizes etc., you can tell `fd` to show them +by running `ls` for each result: +``` bash +fd … -X ls -lhd --color=always +``` +This pattern is so useful that `fd` provides a shortcut. You can use the `-l`/`--list-details` +option to execute `ls` in this way: `fd … -l`. + +The `-X` option is also useful when combining `fd` with [ripgrep](https://github.com/BurntSushi/ripgrep/) (`rg`) in order to search within a certain class of files, like all C++ source files: +```bash +fd -e cpp -e cxx -e h -e hpp -X rg 'std::cout' +``` + +Convert all `*.jpg` files to `*.png` files: +``` bash +fd -e jpg -x convert {} {.}.png +``` +Here, `{}` is a placeholder for the search result. `{.}` is the same, without the file extension. +See below for more details on the placeholder syntax. + +The terminal output of commands run from parallel threads using `-x` will not be interlaced or garbled, +so `fd -x` can be used to rudimentarily parallelize a task run over many files. +An example of this is calculating the checksum of each individual file within a directory. +``` bash +fd -tf -x md5sum > file_checksums.txt +``` + +#### Placeholder syntax + +The `-x` and `-X` options take a *command template* as a series of arguments (instead of a single string). +If you want to add additional options to `fd` after the command template, you can terminate it with a `\;`. + +For example, `fd -x echo \; pattern path` treats `pattern path` as `fd` arguments instead of +passing them to `echo`. In practice, it is often clearer to write `fd pattern path -x echo`. + +The syntax for generating commands is similar to that of [GNU Parallel](https://www.gnu.org/software/parallel/): + +- `{}`: A placeholder token that will be replaced with the path of the search result + (`documents/images/party.jpg`). +- `{.}`: Like `{}`, but without the file extension (`documents/images/party`). +- `{/}`: A placeholder that will be replaced by the basename of the search result (`party.jpg`). +- `{//}`: The parent of the discovered path (`documents/images`). +- `{/.}`: The basename, with the extension removed (`party`). + +If you do not include a placeholder, *fd* automatically adds a `{}` at the end. + +#### Parallel vs. serial execution + +For `-x`/`--exec`, you can control the number of parallel jobs by using the `-j`/`--threads` option. +Use `--threads=1` for serial execution. + +### Excluding specific files or directories + +Sometimes we want to ignore search results from a specific subdirectory. For example, we might +want to search all hidden files and directories (`-H`) but exclude all matches from `.git` +directories. We can use the `-E` (or `--exclude`) option for this. It takes an arbitrary glob +pattern as an argument: +``` bash +> fd -H -E .git … +``` + +We can also use this to skip mounted directories: +``` bash +> fd -E /mnt/external-drive … +``` + +.. or to skip certain file types: +``` bash +> fd -E '*.bak' … +``` + +To make exclude-patterns like these permanent, you can create a `.fdignore` file. They work like +`.gitignore` files, but are specific to `fd`. For example: +``` bash +> cat ~/.fdignore +/mnt/external-drive +*.bak +``` + +> [!NOTE] +> `fd` also supports `.ignore` files that are used by other programs such as `rg` or `ag`. + +If you want `fd` to ignore these patterns globally, you can put them in `fd`'s global ignore file. +This is usually located in `~/.config/fd/ignore` in macOS or Linux, and `%APPDATA%\fd\ignore` in +Windows. + +You may wish to include `.git/` in your `fd/ignore` file so that `.git` directories, and their contents +are not included in output if you use the `--hidden` option. + +### Deleting files + +You can use `fd` to remove all files and directories that are matched by your search pattern. +If you only want to remove files, you can use the `--exec-batch`/`-X` option to call `rm`. For +example, to recursively remove all `.DS_Store` files, run: +``` bash +> fd -H '^\.DS_Store$' -tf -X rm +``` +If you are unsure, always call `fd` without `-X rm` first. Alternatively, use `rm`s "interactive" +option: +``` bash +> fd -H '^\.DS_Store$' -tf -X rm -i +``` + +If you also want to remove a certain class of directories, you can use the same technique. You will +have to use `rm`s `--recursive`/`-r` flag to remove directories. + +> [!NOTE] +> There are scenarios where using `fd … -X rm -r` can cause race conditions: if you have a +path like `…/foo/bar/foo/…` and want to remove all directories named `foo`, you can end up in a +situation where the outer `foo` directory is removed first, leading to (harmless) *"'foo/bar/foo': +No such file or directory"* errors in the `rm` call. + +### Command-line options + +This is the output of `fd -h`. To see the full set of command-line options, use `fd --help` which +also includes a much more detailed help text. + +``` +Usage: fd [OPTIONS] [pattern] [path]... + +Arguments: + [pattern] the search pattern (a regular expression, unless '--glob' is used; optional) + [path]... the root directories for the filesystem search (optional, requires pattern first) + +Options: + -H, --hidden Search hidden files and directories + -I, --no-ignore Do not respect .(git|fd)ignore files + -s, --case-sensitive Case-sensitive search (default: smart case) + -i, --ignore-case Case-insensitive search (default: smart case) + -g, --glob Glob-based search (default: regular expression) + -a, --absolute-path Show absolute instead of relative paths + -l, --list-details Use a long listing format with file metadata + -L, --follow Follow symbolic links + -p, --full-path Search full abs. path (default: filename only) + -d, --max-depth Set maximum search depth (default: none) + -E, --exclude Exclude entries that match the given glob pattern + -t, --type Filter by type: file (f), directory (d/dir), symlink (l), + executable (x), empty (e), socket (s), pipe (p), char-device + (c), block-device (b) + -e, --extension Filter by extension + -S, --size Limit results based on the size of files + --changed-within Filter by file modification time (newer than) + --changed-before Filter by file modification time (older than) + -o, --owner Filter by owning user and/or group + --format Print results according to template + -x, --exec ... Execute a command for each search result + -X, --exec-batch ... Execute a command with all search results at once + -c, --color When to use colors [default: auto] [possible values: auto, + always, never] + --hyperlink[=] Add hyperlinks to output paths [default: never] [possible + values: auto, always, never] + --ignore-contain Ignore directories containing the named entry + -h, --help Print help (see more with '--help') + -V, --version Print version +``` + +Note that options can be given after the pattern and/or path as well. + +## Benchmark + +Let's search my home folder for files that end in `[0-9].jpg`. It contains ~750,000 +subdirectories and about a 4 million files. For averaging and statistical analysis, I'm using +[hyperfine](https://github.com/sharkdp/hyperfine). The following benchmarks are performed +with a "warm"/pre-filled disk-cache (results for a "cold" disk-cache show the same trends). + +Let's start with `find`: +``` +Benchmark 1: find ~ -iregex '.*[0-9]\.jpg$' + Time (mean ± σ): 19.922 s ± 0.109 s + Range (min … max): 19.765 s … 20.065 s +``` + +`find` is much faster if it does not need to perform a regular-expression search: +``` +Benchmark 2: find ~ -iname '*[0-9].jpg' + Time (mean ± σ): 11.226 s ± 0.104 s + Range (min … max): 11.119 s … 11.466 s +``` + +Now let's try the same for `fd`. Note that `fd` performs a regular expression +search by default. The options `-u`/`--unrestricted` option is needed here for +a fair comparison. Otherwise `fd` does not have to traverse hidden folders and +ignored paths (see below): +``` +Benchmark 3: fd -u '[0-9]\.jpg$' ~ + Time (mean ± σ): 854.8 ms ± 10.0 ms + Range (min … max): 839.2 ms … 868.9 ms +``` +For this particular example, `fd` is approximately **23 times faster** than `find -iregex` +and about **13 times faster** than `find -iname`. By the way, both tools found the exact +same 546 files :smile:. + +**Note**: This is *one particular* benchmark on *one particular* machine. While we have +performed a lot of different tests (and found consistent results), things might +be different for you! We encourage everyone to try it out on their own. See +[this repository](https://github.com/sharkdp/fd-benchmarks) for all necessary scripts. + +Concerning *fd*'s speed, a lot of credit goes to the `regex` and `ignore` crates that are +also used in [ripgrep](https://github.com/BurntSushi/ripgrep) (check it out!). + +## Troubleshooting + +### `fd` does not find my file! + +Remember that `fd` ignores hidden directories and files by default. It also ignores patterns +from `.gitignore` files. If you want to make sure to find absolutely every possible file, always +use the options `-u`/`--unrestricted` option (or `-HI` to enable hidden and ignored files): +``` bash +> fd -u … +``` + +Also remember that by default, `fd` only searches based on the filename and +doesn't compare the pattern to the full path. If you want to search based on the +full path (similar to the `-path` option of `find`) you need to use the `--full-path` +(or `-p`) option. + +### Colorized output + +`fd` can colorize files by extension, just like `ls`. In order for this to work, the environment +variable [`LS_COLORS`](https://linux.die.net/man/5/dir_colors) has to be set. Typically, the value +of this variable is set by the `dircolors` command which provides a convenient configuration format +to define colors for different file formats. +On most distributions, `LS_COLORS` should be set already. If you are on Windows or if you are looking +for alternative, more complete (or more colorful) variants, see [here](https://github.com/sharkdp/vivid), +[here](https://github.com/seebi/dircolors-solarized) or +[here](https://github.com/trapd00r/LS_COLORS). + +`fd` also honors the [`NO_COLOR`](https://no-color.org/) environment variable. + +### `fd` doesn't seem to interpret my regex pattern correctly + +A lot of special regex characters (like `[]`, `^`, `$`, ..) are also special characters in your +shell. If in doubt, always make sure to put single quotes around the regex pattern: + +``` bash +> fd '^[A-Z][0-9]+$' +``` + +If your pattern starts with a dash, you have to add `--` to signal the end of command line +options. Otherwise, the pattern will be interpreted as a command-line option. Alternatively, +use a character class with a single hyphen character: + +``` bash +> fd -- '-pattern' +> fd '[-]pattern' +``` + +### "Command not found" for `alias`es or shell functions + +Shell `alias`es and shell functions can not be used for command execution via `fd -x` or +`fd -X`. In `zsh`, you can make the alias global via `alias -g myalias="…"`. In `bash`, +you can use `export -f my_function` to make available to child processes. You would still +need to call `fd -x bash -c 'my_function "$1"' bash`. For other use cases or shells, use +a (temporary) shell script. + +### Placeholders in `-x`/`-X` + +Depending on your shell, you may need to quote the placeholders (`{}`, `{/}`, `{//}`, +`{.}`, `{/.}`) to prevent the shell from interpreting them before `fd` sees them. + +### `fd` crashes when another memory allocator is preloaded + +The released Linux binaries use jemalloc, through the `use-jemalloc` feature, which is enabled by default for +some targets. Preloading a different allocator while jemalloc is compiled in is not supported, for example + +``` bash +> LD_PRELOAD=/usr/bin/mimalloc fd +``` + +makes the two disagree about which of them owns an allocation, and `fd` crashes. `fd` cannot detect +this and report it, because the preloaded library is already answering its first allocation. To run +with a preloaded allocator, build without jemalloc: + +``` bash +> cargo build --release --no-default-features +``` + +## Integration with other programs + +### Using fd with `fzf` + +You can use *fd* to generate input for the command-line fuzzy finder [fzf](https://github.com/junegunn/fzf): +``` bash +export FZF_DEFAULT_COMMAND='fd --type file' +export FZF_CTRL_T_COMMAND="$FZF_DEFAULT_COMMAND" +``` + +Then, you can type `vim ` on your terminal to open fzf and search through the fd-results. + +Alternatively, you might like to follow symbolic links and include hidden files (but exclude `.git` folders): +``` bash +export FZF_DEFAULT_COMMAND='fd --type file --follow --hidden --exclude .git' +``` + +You can even use fd's colored output inside fzf by setting: +``` bash +export FZF_DEFAULT_COMMAND="fd --type file --color=always" +export FZF_DEFAULT_OPTS="--ansi" +``` + +For more details, see the [Tips section](https://github.com/junegunn/fzf#tips) of the fzf README. + +### Using fd with `rofi` + +[*rofi*](https://github.com/davatorium/rofi) is a graphical launch menu application that is able to create menus by reading from *stdin*. Piping `fd` output into `rofi`s `-dmenu` mode creates fuzzy-searchable lists of files and directories. + +#### Example + +Create a case-insensitive searchable multi-select list of *PDF* files under your `$HOME` directory and open the selection with your configured PDF viewer. To list all file types, drop the `-e pdf` argument. + +``` bash +fd --type f -e pdf . $HOME | rofi -keep-right -dmenu -i -p FILES -multi-select | xargs -I {} xdg-open {} +``` + +To modify the list that is presented by rofi, add arguments to the `fd` command. To modify the search behaviour of rofi, add arguments to the `rofi` command. + +### Using fd with `emacs` + +The emacs package [find-file-in-project](https://github.com/technomancy/find-file-in-project) can +use *fd* to find files. + +After installing `find-file-in-project`, add the line `(setq ffip-use-rust-fd t)` to your +`~/.emacs` or `~/.emacs.d/init.el` file. + +In emacs, run `M-x find-file-in-project-by-selected` to find matching files. Alternatively, run +`M-x find-file-in-project` to list all available files in the project. + +### Printing the output as a tree + +To format the output of `fd` as a file-tree you can use the `tree` command with +`--fromfile`: +```bash +⯠fd | tree --fromfile +``` + +This can be more useful than running `tree` by itself because `tree` does not +ignore any files by default, nor does it support as rich a set of options as +`fd` does to control what to print: +```bash +⯠fd --extension rs | tree --fromfile +. +├── build.rs +└── src + ├── app.rs + └── error.rs +``` + +On bash and similar you can simply create an alias: +```bash +⯠alias as-tree='tree --fromfile' +``` + +### Using fd with `xargs` or `parallel` + +Note that `fd` has a builtin feature for [command execution](#command-execution) with +its `-x`/`--exec` and `-X`/`--exec-batch` options. If you prefer, you can still use +it in combination with `xargs`: +``` bash +> fd -0 -e rs | xargs -0 wc -l +``` +Here, the `-0` option tells *fd* to separate search results by the NULL character (instead of +newlines). In the same way, the `-0` option of `xargs` tells it to read the input in this way. + +## Installation + +[![Packaging status](https://repology.org/badge/vertical-allrepos/fd-find.svg)](https://repology.org/project/fd-find/versions) + +### On Ubuntu +*... and other Debian-based Linux distributions.* + +If you run Ubuntu 19.04 (Disco Dingo) or newer, you can install the +[officially maintained package](https://packages.ubuntu.com/fd-find): +``` +apt install fd-find +``` +Note that the binary is called `fdfind` as the binary name `fd` is already used by another package. +It is recommended that after installation, you add a link to `fd` by executing command +`ln -s $(which fdfind) ~/.local/bin/fd`, in order to use `fd` in the same way as in this documentation. +Make sure that `$HOME/.local/bin` is in your `$PATH`. + +If you use an older version of Ubuntu, you can download the latest `.deb` package from the +[release page](https://github.com/sharkdp/fd/releases) and install it via: +``` bash +dpkg -i fd_9.0.0_amd64.deb # adapt version number and architecture +``` + +Note that the .deb packages on the release page for this project still name the executable `fd`. + +### On Debian + +If you run Debian Buster or newer, you can install the +[officially maintained Debian package](https://tracker.debian.org/pkg/rust-fd-find): +``` +apt-get install fd-find +``` +Note that the binary is called `fdfind` as the binary name `fd` is already used by another package. +It is recommended that after installation, you add a link to `fd` by executing command +`ln -s $(which fdfind) ~/.local/bin/fd`, in order to use `fd` in the same way as in this documentation. +Make sure that `$HOME/.local/bin` is in your `$PATH`. + +Note that the .deb packages on the release page for this project still name the executable `fd`. + +### On Fedora + +Starting with Fedora 28, you can install `fd` from the official package sources: +``` bash +dnf install fd-find +``` + +### On Alpine Linux + +You can install [the fd package](https://pkgs.alpinelinux.org/packages?name=fd) +from the official sources, provided you have the appropriate repository enabled: +``` +apk add fd +``` + +### On Arch Linux + +You can install [the fd package](https://www.archlinux.org/packages/extra/x86_64/fd/) from the official repos: +``` +pacman -S fd +``` +You can also install fd [from the AUR](https://aur.archlinux.org/packages/fd-git). + +### On Gentoo Linux + +You can use [the fd ebuild](https://packages.gentoo.org/packages/sys-apps/fd) from the official repo: +``` +emerge -av fd +``` + +### On openSUSE Linux + +You can install [the fd package](https://software.opensuse.org/package/fd) from the official repo: +``` +zypper in fd +``` + +### On Void Linux + +You can install `fd` via xbps-install: +``` +xbps-install -S fd +``` + +### On ALT Linux + +You can install [the fd package](https://packages.altlinux.org/en/sisyphus/srpms/fd/) from the official repo: +``` +apt-get install fd +``` + +### On Solus + +You can install [the fd package](https://github.com/getsolus/packages/tree/main/packages/f/fd) from the official repo: +``` +eopkg install fd +``` + +### On RedHat Enterprise Linux (RHEL) 8/9/10, Almalinux 8/9/10, EuroLinux 8/9 or Rocky Linux 8/9/10 + +You can install [the `fd` package](https://copr.fedorainfracloud.org/coprs/tkbcopr/fd/) from Fedora Copr. + +```bash +dnf copr enable tkbcopr/fd +dnf install fd +``` + +A different version using the [slower](https://github.com/sharkdp/fd/pull/481#issuecomment-534494592) malloc [instead of jemalloc](https://bugzilla.redhat.com/show_bug.cgi?id=2216193#c1) is also available from the EPEL8/9 repo as the package `fd-find`. + +### On macOS + +You can install `fd` with [Homebrew](https://formulae.brew.sh/formula/fd): +``` +brew install fd +``` + +… or with MacPorts: +``` +port install fd +``` + +### On Windows + +You can download pre-built binaries from the [release page](https://github.com/sharkdp/fd/releases). + +Alternatively, you can install `fd` via [Scoop](http://scoop.sh): +``` +scoop install fd +``` + +Or via [Chocolatey](https://chocolatey.org): +``` +choco install fd +``` + +Or via [Winget](https://learn.microsoft.com/en-us/windows/package-manager/): +``` +winget install sharkdp.fd +``` + +### On GuixOS + +You can install [the fd package](https://guix.gnu.org/en/packages/fd-8.1.1/) from the official repo: +``` +guix install fd +``` + +### On Mise + +You can use [mise](https://github.com/jdx/mise) to install `fd` with a command like this: +``` +mise use -g fd@latest +``` + +### On NixOS / via Nix + +You can use the [Nix package manager](https://nixos.org/nix/) to install `fd`: +``` +nix-env -i fd +``` + +### Via Flox + +You can use [Flox](https://flox.dev) to install `fd` into a Flox environment: +``` +flox install fd +``` + +### On FreeBSD + +You can install [the fd-find package](https://www.freshports.org/sysutils/fd) from the official repo: +``` +pkg install fd-find +``` + +### From npm + +On Linux and macOS, you can install the [fd-find](https://npm.im/fd-find) package: + +``` +npm install -g fd-find +``` + +### From source + +With Rust's package manager [cargo](https://github.com/rust-lang/cargo), you can install *fd* via: +``` +cargo install fd-find +``` +Note that rust version *1.90.0* or later is required. + +`make` is also needed for the build. + +### From binaries + +The [release page](https://github.com/sharkdp/fd/releases) includes precompiled binaries for Linux, macOS and Windows. Statically-linked binaries are also available: look for archives with `musl` in the file name. + +## Development +```bash +git clone https://github.com/sharkdp/fd + +# Build +cd fd +cargo build + +# Run unit tests and integration tests +cargo test + +# Install +cargo install --path . +``` + +### Completions + +#### From Release Archives + +Pre-built completion files are included in the release archives (`.tar.gz`/`.zip`) on the +[Releases page](https://github.com/sharkdp/fd/releases), in the `autocomplete` directory. +To use these completions: + +- **bash**: Source the `fd.bash` file in your `~/.bashrc`, or place it in a directory that gets sourced automatically. +- **zsh**: Move `_fd` to a directory in your `fpath` (e.g., `~/.zfunc`). +- **fish**: Copy `fd.fish` to `~/.config/fish/completions/`. +- **powershell**: Source `_fd.ps1` from one of your [profile scripts](https://learn.microsoft.com/en-us/powershell/scripting/learn/shell/creating-profiles?view=powershell-7.5). + +#### Generate from fd + +You can also generate completions directly using `fd --gen-completions `: + +```bash +# Bash +fd --gen-completions bash > ~/.local/share/bash-completion/completions/fd + +# Zsh (ensure ~/.zfunc is in your fpath) +fd --gen-completions zsh > ~/.zfunc/_fd + +# Fish +fd --gen-completions fish > ~/.config/fish/completions/fd.fish + +# PowerShell +fd --gen-completions powershell >> $PROFILE +``` + +## Maintainers + +- [sharkdp](https://github.com/sharkdp) +- [tmccombs](https://github.com/tmccombs) +- [tavianator](https://github.com/tavianator) + +## License + +`fd` is distributed under the terms of both the MIT License and the Apache License 2.0. + +See the [LICENSE-APACHE](LICENSE-APACHE) and [LICENSE-MIT](LICENSE-MIT) files for license details. diff --git a/userland/fd/SECURITY.md b/userland/fd/SECURITY.md new file mode 100644 index 0000000..eecd288 --- /dev/null +++ b/userland/fd/SECURITY.md @@ -0,0 +1,33 @@ +# Security Reporting + +If you wish to report a security vulnerability privately, we appreciate your diligence. Please follow the guidelines below to submit your report. + +## Reporting + +To report a security vulnerability, please provide the following information: + +1. **PROJECT** + - Include the URL of the project repository - Example: + +2. **PUBLIC** + - Indicate whether this vulnerability has already been publicly discussed or disclosed. + - If so, provide relevant links. + +3. **DESCRIPTION** + - Provide a detailed description of the security vulnerability. + - Include as much information as possible to help us understand and address the issue. + +Send this information, along with any additional relevant details, to our [vulnerability reporting form](https://github.com/sharkdp/fd/security/advisories/new). + +## Confidentiality + +We kindly ask you to keep the report confidential until a public announcement is made. + +## Notes + +- Vulnerabilities will be handled on a best-effort basis. +- You may request an advance copy of the patched release, but we cannot guarantee early access before the public release. +- You will be notified via email simultaneously with the public announcement. +- We will respond within a few weeks to confirm whether your report has been accepted or rejected. + +Thank you for helping to improve the security of our project! diff --git a/userland/fd/contrib/completion/_fd b/userland/fd/contrib/completion/_fd new file mode 100644 index 0000000..5ee9e20 --- /dev/null +++ b/userland/fd/contrib/completion/_fd @@ -0,0 +1,299 @@ +#compdef fd + +## +# zsh completion function for fd +# +# Based on ripgrep completion function. +# Originally based on code from the zsh-users project — see copyright notice +# below. + +autoload -U is-at-least + +_fd() { + local curcontext="$curcontext" no='!' ret=1 + local -a context line state state_descr _arguments_options fd_types fd_args + local -A opt_args + + if is-at-least 5.2; then + _arguments_options=( -s -S ) + else + _arguments_options=( -s ) + fi + + fd_types=( + {f,file}'\:"regular files"' + {d,dir,directory}'\:"directories"' + {l,symlink}'\:"symbolic links"' + {e,empty}'\:"empty files or directories"' + {x,executable}'\:"executable (files)"' + {b,block-device}'\:"block devices"' + {c,char-device}'\:"character devices"' + {s,socket}'\:"sockets"' + {p,pipe}'\:"named pipes (FIFOs)"' + ) + + # Do not complete rare options unless either the current prefix + # matches one of those options or the user has the `complete-all` + # style set. Note that this prefix check has to be updated manually to account + # for all of the potential negation options listed below! + if + # (--[bpsu]* => match all options marked with '$no') + [[ $PREFIX$SUFFIX == --[bopsun]* ]] || + zstyle -t ":complete:$curcontext:*" complete-all + then + no= + fi + + # We make heavy use of argument groups here to prevent the option specs from + # growing unwieldy. These aren't supported in zsh <5.4, though, so we'll strip + # them out below if necessary. This makes the exclusions inaccurate on those + # older versions, but oh well — it's not that big a deal + fd_args=( + + '(hidden)' # hidden files + {-H,--hidden}'[search hidden files/directories]' + + + '(no-ignore-full)' # all ignore files + '(no-ignore-partial)'{-I,--no-ignore}"[don't respect .(git|fd)ignore and global ignore files]" + $no'(no-ignore-partial)*'{-u,--unrestricted}'[alias for --no-ignore --hidden]' + + + no-ignore-partial # some ignore files + "(no-ignore-full --no-ignore-vcs)--no-ignore-vcs[don't respect .gitignore files]" + "!(no-ignore-full --no-global-ignore-file)--no-global-ignore-file[don't respect the global ignore file]" + "$no(no-ignore-full --no-ignore-parent)--no-ignore-parent[don't respect .(git|fd)ignore files in parent directories]" + + + '(case)' # case-sensitivity + {-s,--case-sensitive}'[perform a case-sensitive search]' + {-i,--ignore-case}'[perform a case-insensitive search]' + + + '(regex-pattern)' # regex-based search pattern + '(no-regex-pattern)--regex[perform a regex-based search (default)]' + + + '(no-regex-pattern)' # non-regex-based search pattern + {-g,--glob}'[perform a glob-based search]' + {-F,--fixed-strings}'[treat pattern as literal string instead of a regex]' + '!--literal' + '--exact[pattern must exactly match the full filename]' + + + '(no-require-git)' + "$no(no-ignore-full --no-ignore-vcs --no-require-git)--no-require-git[don't require git repo to respect gitignores]" + + + '(match-full)' # match against full path + {-p,--full-path}'[match the pattern against the full path instead of the basename]' + + + '(follow)' # follow symlinks + {-L,--follow}'[follow symbolic links to directories]' + '!--dereference' + + + '(abs-path)' # show absolute paths + '(long-listing)'{-a,--absolute-path}'[show absolute paths instead of relative paths]' + + + '(null-sep)' # use null separator for output + '(long-listing)'{-0,--print0}'[separate search results by the null character]' + + + '(long-listing)' # long-listing output + '(abs-path null-sep max-results exec-cmds)'{-l,--list-details}'[use a long listing format with file metadata]' + + + '(output-format)' # template-based output + '(long-listing)'--format=[print results according to template]:format + + + '(max-results)' # max number of results + '(long-listing exec-cmds quiet)--max-results=[limit number of search results to given count and quit]:count' + '(long-listing exec-cmds)-1[limit to a single search result and quit]' + + + '(quiet)' + '(max-results)'{-q,--quiet}'[print nothing. exit code 1 if no matches]' + '!(max-results)--has-results' + + + '(fs-errors)' # file-system errors + $no'--show-errors[enable the display of filesystem errors]' + + + '(fs-traversal)' # file-system traversal + $no"--one-file-system[don't descend into directories on other file systems]" + '!--mount' + '!--xdev' + + + dir-depth # directory depth + '(--exact-depth -d --max-depth)'{-d+,--max-depth=}'[set max directory depth to descend when searching]:depth' + '!(--exact-depth -d --max-depth)--maxdepth=:depth' + '(--exact-depth --min-depth)--min-depth=[set directory depth to descend before start searching]:depth' + '!(--exact-depth --min-depth)--mindepth=:depth' + '(--exact-depth -d --max-depth --maxdepth --min-depth)--exact-depth=[only search at the exact given directory depth]:depth' + + + prune # pruning + "--prune[don't traverse into matching directories]" + + + filter-misc # filter search + '*'{-t+,--type=}"[filter search by type]:type:(($fd_types))" + '*'{-e+,--extension=}'[filter search by file extension]:extension' + '*'{-E+,--exclude=}'[exclude files/directories that match the given glob pattern]:glob pattern' + '*'{-S+,--size=}'[limit search by file size]:size limit:->size' + '(-o --owner)'{-o+,--owner=}'[filter by owning user and/or group]:owner and/or group:->owner' + + + ignore-file # extra ignore files + '*--ignore-file=[add a custom, low-precedence ignore-file with .gitignore format]: :_files' + + + '(filter-mtime-newer)' # filter by files modified after than + '--changed-within=[limit search to files/directories modified within the given date/duration]:date or duration' + '--changed-after=[alias for --changed-within]:date/duration' + '!--change-newer-than=:date/duration' + '!--newer=:date/duration' + + + '(filter-mtime-older)' # filter by files modified before than + '--changed-before=[limit search to files/directories modified before the given date/duration]:date or duration' + '!--change-older-than=:date/duration' + '!--older=:date/duration' + + + '(color)' # colorize output + {-c+,--color=}'[declare when to colorize search results]:when to colorize:(( + auto\:"show colors if the output goes to an interactive console (default)" + never\:"do not use colorized output" + always\:"always use colorized output" + ))' + + '--hyperlink=-[add hyperlinks to output paths]::when:(auto never always)' + '!--hyper=-' + + + '(threads)' + {-j+,--threads=}'[set the number of threads for searching and executing]:number of threads' + + + '(exec-cmds)' # execute command + '(long-listing max-results)'{-x+,--exec=}'[execute command for each search result]:command: _command_names -e:*\;::program arguments: _normal' + '(long-listing max-results)'{-X+,--exec-batch=}'[execute command for all search results at once]:command: _command_names -e:*\;::program arguments: _normal' + '(long-listing max-results)--batch-size=[max number of args for each -X call]:size' + + + other + '!(--max-buffer-time)--max-buffer-time=[set amount of time to buffer before showing output]:time (ms)' + + '*--ignore-contain=[Ignore dirs containing named file]:name' + + + '(about)' # about flags + '(: * -)'{-h,--help}'[display help message]' + '(: * -)'{-V,--version}'[display version information]' + + + path-sep # set path separator for output + $no'(--path-separator)--path-separator=[set the path separator to use when printing file paths]:path separator' + + + search-path + $no'(--base-directory)--base-directory=[change the current working directory to the given path]:directory:_files -/' + $no'(*)*--search-path=[set search path (instead of positional arguments)]:directory:_files -/' + + + strip-cwd-prefix + $no'(strip-cwd-prefix exec-cmds)--strip-cwd-prefix=-[When to strip ./]::when:(always never auto)' + + + and + '*--and=[additional required search pattern]:pattern' + + + + args # positional arguments + '1: :_guard "^-*" pattern' + '(--search-path)*:directory:_files -/' + ) + + # Strip out argument groups where unsupported (see above) + is-at-least 5.4 || + fd_args=( ${(@)args:#(#i)(+|[a-z0-9][a-z0-9_-]#|\([a-z0-9][a-z0-9_-]#\))} ) + + _arguments $_arguments_options : $fd_args && ret=0 + + case ${state} in + owner) + compset -P '(\\|)\!' + if compset -P '*:'; then + _groups && ret=0 + else + if + compset -S ':*' || + # Do not add the colon suffix when completing "!user + # (with a starting double-quote) otherwise pressing tab again + # after the inserted colon "!user: will complete history modifiers + [[ $IPREFIX == (\\|\!)* && ($QIPREFIX == \"* && -z $QISUFFIX) ]] + then + _users && ret=0 + else + local q + # Since quotes are needed when using the negation prefix !, + # automatically remove the colon suffix also when closing the quote + if [[ $QIPREFIX == [\'\"]* ]]; then + q=${QIPREFIX:0:1} + fi + _users -r ": \t\n\-$q" -S : && ret=0 + fi + fi + ;; + + size) + if compset -P '[-+][0-9]##'; then + local -a suff=( + 'B:bytes' + 'K:kilobytes (10^3 = 1000 bytes)' + 'M:megabytes (10^6 = 1000^2 bytes)' + 'G:gigabytes (10^9 = 1000^3 bytes)' + 'T:terabytes (10^12 = 1000^4 bytes)' + 'Ki:kibibytes ( 2^10 = 1024 bytes)' + 'Mi:mebibytes ( 2^20 = 1024^2 bytes)' + 'Gi:gibibytes ( 2^30 = 1024^3 bytes)' + 'Ti:tebibytes ( 2^40 = 1024^4 bytes)' + ) + _describe -t units 'size limit units' suff -V 'units' + elif compset -P '[-+]'; then + _message -e 'size limit number (full format: <+->)' + else + _values 'size limit prefix (full format: )' \ + '\+[file size must be greater or equal to]'\ + '-[file size must be less than or equal to]' && ret=0 + fi + ;; + esac + + return ret +} + +_fd "$@" + +# ------------------------------------------------------------------------------ +# Copyright (c) 2011 GitHub zsh-users - http://github.com/zsh-users +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of the zsh-users nor the +# names of its contributors may be used to endorse or promote products +# derived from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL ZSH-USERS BE LIABLE FOR ANY +# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# ------------------------------------------------------------------------------ +# Description +# ----------- +# +# Completion script for fd +# +# ------------------------------------------------------------------------------ +# Authors +# ------- +# +# * smancill (https://github.com/smancill) +# +# ------------------------------------------------------------------------------ + +# Local Variables: +# mode: shell-script +# coding: utf-8-unix +# indent-tabs-mode: nil +# sh-indentation: 2 +# sh-basic-offset: 2 +# End: +# vim: ft=zsh sw=2 ts=2 et diff --git a/userland/fd/contrib/completion/_fdfind b/userland/fd/contrib/completion/_fdfind new file mode 100644 index 0000000..3071f03 --- /dev/null +++ b/userland/fd/contrib/completion/_fdfind @@ -0,0 +1,2 @@ +#compdef fdfind +_fd "$@" diff --git a/userland/fd/contrib/completion/fdfind.bash b/userland/fd/contrib/completion/fdfind.bash new file mode 100644 index 0000000..d5e05b6 --- /dev/null +++ b/userland/fd/contrib/completion/fdfind.bash @@ -0,0 +1,8 @@ +# Source fd completions +source /usr/share/bash-completion/completions/fd + +if [[ "${BASH_VERSINFO[0]}" -eq 4 && "${BASH_VERSINFO[1]}" -ge 4 || "${BASH_VERSINFO[0]}" -gt 4 ]]; then + complete -F _fd -o nosort -o bashdefault -o default fdfind +else + complete -F _fd -o bashdefault -o default fdfind +fi diff --git a/userland/fd/contrib/completion/fdfind.fish b/userland/fd/contrib/completion/fdfind.fish new file mode 100644 index 0000000..b435f3c --- /dev/null +++ b/userland/fd/contrib/completion/fdfind.fish @@ -0,0 +1 @@ +complete -c fdfind --wraps fd diff --git a/userland/fd/doc/.gitattributes b/userland/fd/doc/.gitattributes new file mode 100644 index 0000000..36eaad9 --- /dev/null +++ b/userland/fd/doc/.gitattributes @@ -0,0 +1 @@ +* linguist-vendored diff --git a/userland/fd/doc/fd.1 b/userland/fd/doc/fd.1 new file mode 100644 index 0000000..3a260ef --- /dev/null +++ b/userland/fd/doc/fd.1 @@ -0,0 +1,588 @@ +.TH FD 1 +.SH NAME +fd \- find entries in the filesystem +.SH SYNOPSIS +.B fd +.RB [ \-HIEsiaLp0hV ] +.RB [ \-d +.IR depth ] +.RB [ \-t +.IR filetype ] +.RB [ \-e +.IR ext ] +.RB [ \-E +.IR exclude ] +.RB [ \-c +.IR when ] +.RB [ \-j +.IR num ] +.RB [ \-x +.IR cmd ] +.RI [ pattern +.RI [ path... ]] +.SH DESCRIPTION +.B fd +is a simple, fast and user-friendly alternative to +.BR find (1). +.P +By default +.B fd +uses regular expressions for the pattern. However, this can be changed to use simple glob patterns +with the '\-\-glob' option. +.P +By default +.B fd +will exclude hidden files and directories, as well as any files that match gitignore rules +or ignore rules in .ignore or .fdignore files. +.P +If you wish to search all files in a specific directory, you can use '' or . as the search pattern, +to match all files. Or you can use the '\-\-search\-path' option to specify the path(s) instead of +the positional parameter. +.P +Options may be given anywhere on the command line. +.SH OPTIONS +.TP +.B \-H, \-\-hidden +Include hidden files and directories in the search results +(default: hidden files and directories are skipped). The flag can be overridden with '--no-hidden'. +.IP +Ignored files are still excluded unless \-\-no\-ignore or \-\-no\-ignore\-vcs +is also used. +.TP +.B \-I, \-\-no\-ignore +Show search results from files and directories that would otherwise be ignored by +.RS +.IP \[bu] 2 +.I .gitignore +.IP \[bu] +.I .git/info/exclude +.IP \[bu] +The global gitignore configuration (by default +.IR $HOME/.config/git/ignore ) +.IP \[bu] +.I .ignore +.IP \[bu] +.I .fdignore +.IP \[bu] +The global fd ignore file (usually +.I $HOME/.config/fd/ignore +) +.RE +.IP +The flag can be overridden with '--ignore'. +.TP +.B \-u, \-\-unrestricted +Perform an unrestricted search, including ignored and hidden files. This is an alias for '--hidden --no-ignore'. +.TP +.B \-\-no\-ignore\-vcs +Show search results from files and directories that would otherwise be ignored by gitignore files +including +.IR .gitignore , +.IR .git/info/exclude , +and the global gitignore configuration +.RI ( core.excludesFile +git setting, which defaults to +.IR $HOME/.config/git/ignore ). +The flag can be overridden with '--ignore-vcs'. +.TP +.B \-\-no\-require\-git +Do not require a git repository to respect gitignores. By default, fd will only +respect global gitignore rules, .gitignore rules and local exclude rules if fd +detects that you are searching inside a git repository. This flag allows you to +relax this restriction such that fd will respect all git related ignore rules +regardless of whether you’re searching in a git repository or not. The flag can +be overridden with '--require-git'. +.TP +.B \-\-no\-ignore\-parent +Show search results from files and directories that would otherwise be ignored by +'.gitignore', '.ignore', or '.fdignore' files in parent directories. +.TP +.B \-s, \-\-case\-sensitive +Perform a case-sensitive search. By default, fd uses case-insensitive searches, unless the +pattern contains an uppercase character (smart case). +.TP +.B \-i, \-\-ignore\-case +Perform a case-insensitive search. By default, fd uses case-insensitive searches, unless the +pattern contains an uppercase character (smart case). +.TP +.B \-g, \-\-glob +Perform a glob-based search instead of a regular expression search. +If combined with the '\-\-full-path' option, '**' can be used to match multiple path components. +.TP +.B \-\-regex +Perform a regular-expression based search (default). This can be used to override --glob. +.TP +.B \-F, \-\-fixed\-strings +Treat the pattern as a literal string instead of a regular expression. Note that this also +performs substring comparison. If you want to match on an exact filename, consider using '\-\-glob' +or '\-\-exact'. +.TP +.B \-\-exact +Perform an exact match. This is equivalent to '\-\-fixed\-strings' but requires the pattern to +match the entire filename (or the full path if '\-\-full\-path' is used), rather than a substring. +Special regex characters in the pattern are treated as literal characters. +Cannot be used together with '\-\-glob' or '\-\-fixed\-strings'. +.TP +.BI "\-\-and " pattern +Add additional required search patterns, all of which must be matched. Multiple additional +patterns can be specified. The patterns are regular expressions, unless '\-\-glob' +or '\-\-fixed\-strings' is used. +.TP +.B \-a, \-\-absolute\-path +Shows the full path starting from the root as opposed to relative paths. +The flag can be overridden with '--relative-path'. +.TP +.B \-l, \-\-list\-details +Use a detailed listing format like 'ls -l'. This is basically an alias +for '--exec-batch ls -l' with some additional 'ls' options. This can be used +to see more metadata, to show symlink targets and to achieve a deterministic +sort order. +.TP +.B \-L, \-\-follow +By default, fd does not descend into symlinked directories. Using this flag, symbolic links are +also traversed. The flag can be overridden with '--no-follow'. +.TP +.B \-p, \-\-full\-path +By default, the search pattern is only matched against the filename (or directory name). Using +this flag, the +.I pattern +is matched against the full path. +.TP +.B \-0, \-\-print0 +Separate search results by the null character (instead of newlines). Useful for piping results to +.IR xargs . +.TP +.B \-\-max\-results count +Limit the number of search results to 'count' and quit immediately. +.TP +.B \-1 +Limit the search to a single result and quit immediately. This is an alias for '--max-results=1'. +.TP +.B \-q, \-\-quiet +When the flag is present, the program does not print anything and will instead exit with a code of 0 if there is at least one search result. +Otherwise, the exit code will be 1. +This is mainly for usage in scripts and can be faster than checking for output because the search can be stopped early after the first match. +.B \-\-has\-results +can be used as an alias. +.TP +.B \-\-show-errors +Enable the display of filesystem errors for situations such as insufficient +permissions or dead symlinks. +.TP +.B \-\-strip-cwd-prefix [when] +By default, relative paths are prefixed with './' when -x/--exec, +-X/--exec-batch, or -0/--print0 are given, to reduce the risk of a +path starting with '-' being treated as a command line option. Use +this flag to change this behavior. If this flag is used without a value, +it is equivalent to passing "always". Possible values are: +.RS +.IP never +Never strip the ./ at the beginning of paths +.IP always +Always strip the ./ at the beginning of paths +.IP auto +Only strip if used with --exec, --exec-batch, or --print0. That is, it resets to the default behavior. +.RE +.TP +.B \-\-one\-file\-system, \-\-mount, \-\-xdev +By default, fd will traverse the file system tree as far as other options dictate. With this flag, fd ensures that it does not descend into a different file system than the one it started in. Comparable to the -mount or -xdev filters of find(1). +.TP +.B \-h, \-\-help +Print help information. +.TP +.B \-V, \-\-version +Print version information. +.TP +.BI "\-d, \-\-max\-depth " d +Limit directory traversal to at most +.I d +levels of depth. By default, there is no limit on the search depth. +.TP +.BI "\-\-min\-depth " d +Only show search results starting at the given depth. See also: '--max-depth' and '--exact-depth'. +.TP +.BI "\-\-exact\-depth " d +Only show search results at the exact given depth. This is an alias for '--min-depth --max-depth '. +.TP +.B \-\-prune +Do not traverse into matching directories. +.TP +.BI "\-t, \-\-type " filetype +Filter search by type: +.RS +.IP "f, file" 20 +regular files +.IP "d, dir, directory" +directories +.IP "l, symlink" +symbolic links +.IP "b, block-device" +block devices +.IP "c, char-device" +character devices +.IP "s, socket" +sockets +.IP "p, pipe" +named pipes (FIFOs) +.IP "x, executable" +executable (files) +.IP "e, empty" +empty files or directories +.RE + +.RS +This option can be specified more than once to include multiple file types. +Searching for '--type file --type symlink' will show both regular files as well as +symlinks. Note that the 'executable' and 'empty' filters work differently: '--type +executable' implies '--type file' by default. And '--type empty' searches for +empty files and directories, unless either '--type file' or '--type directory' is +specified in addition. + +Examples: + - Only search for files: + fd --type file … + fd -tf … + - Find both files and symlinks + fd --type file --type symlink … + fd -tf -tl … + - Find executable files: + fd --type executable + fd -tx + - Find empty files: + fd --type empty --type file + fd -te -tf + - Find empty directories: + fd --type empty --type directory + fd -te -td +.RE +.TP +.BI "\-e, \-\-extension " ext +Filter search results by extension +.IR ext . +This option can be used repeatedly to allow for multiple possible extensions. +By default, extension filters apply to both files and directories. Use +\fB\-\-type\fR to restrict the search to specific types. + +If you want to search for files without extension, you can use the regex '^[^.]+$' +as a normal search pattern. +.TP +.BI "\-E, \-\-exclude " glob +Exclude files/directories that match the given glob pattern. +This overrides any other ignore logic. +Multiple exclude patterns can be specified. +Examples: + \-\-exclude '*.pyc' + \-\-exclude node_modules +.TP +.BI "\-\-ignore-contain " name +Exclude directories that (directly) contain the given name. +This option can be specified multiple times. +.TP +.BI "\-\-ignore-file " path +Add a custom ignore-file in '.gitignore' format. +These files have a low precedence. +.TP +.BI "\-c, \-\-color " when +Declare +.I when +to colorize search results: +.RS +.IP auto +Colorize output when standard output is connected to terminal (default). +.IP never +Do not colorize output. +.IP always +Always colorize output. +.RE +.TP +.B "\-\-hyperlink +Specify whether the output should use terminal escape codes to indicate a hyperlink to a +file url pointing to the path. + +The value can be auto, always, or never. + +Currently, the default is "never", and if the option is used without an argument "auto" is +used. In the future this may be changed to "auto" and "always". +.RS +.IP auto +Only output hyperlinks if color is also enabled, as a proxy for whether terminal escape +codes are acceptable. +.IP never +Never output hyperlink escapes. +.IP always +Always output hyperlink escapes, regardless of color settings. +.RE +.TP +.BI "\-j, \-\-threads " num +Set number of threads to use for searching & executing (default: number of available CPU cores). +.TP +.BI "\-S, \-\-size " size +Limit results based on the size of files using the format +.I <+-> +.RS +.IP '+' +file size must be greater than or equal to this +.IP '-' +file size must be less than or equal to this +.P +If neither '+' nor '-' is specified, file size must be exactly equal to this. +.IP 'NUM' +The numeric size (e.g. 500) +.IP 'UNIT' +The units for NUM. They are not case-sensitive. +Allowed unit values: +.RS +.IP 'b' +bytes +.IP 'k' +kilobytes (base ten, 10^3 = 1000 bytes) +.IP 'm' +megabytes +.IP 'g' +gigabytes +.IP 't' +terabytes +.IP 'ki' +kibibytes (base two, 2^10 = 1024 bytes) +.IP 'mi' +mebibytes +.IP 'gi' +gibibytes +.IP 'ti' +tebibytes +.RE +.RE +.TP +.BI "\-\-changed-within " date|duration +Filter results based on the file modification time. +Files with modification timestamps greater than the argument will be returned. +The argument can be provided as a duration (\fI10h, 1d, 35min\fR) or as a specific point +in time as full RFC3339 format with time zone, as a date or datetime in the +local time zone (\fIYYYY-MM-DD\fR or \fIYYYY-MM-DD HH:MM:SS\fR), or as the prefix '@' +followed by the number of seconds since the Unix epoch (@[0-9]+). +\fB\-\-change-newer-than\fR, +.B --newer +or +.B --changed-after +can be used as aliases. + +Examples: + \-\-changed-within 2weeks + \-\-change-newer-than "2018-10-27 10:00:00" + \-\-newer 2018-10-27 + \-\-changed-after @1704067200 +.TP +.BI "\-\-changed-before " date|duration +Filter results based on the file modification time. +Files with modification timestamps less than the argument will be returned. +The argument can be provided as a duration (\fI10h, 1d, 35min\fR) or as a specific point +in time as full RFC3339 format with time zone, as a date or datetime in the +local time zone (\fIYYYY-MM-DD\fR or \fIYYYY-MM-DD HH:MM:SS\fR), or as the prefix '@' +followed by the number of seconds since the Unix epoch (@[0-9]+). +.B --change-older-than +or +.B --older +can be used as aliases. + +Examples: + \-\-changed-before "2018-10-27 10:00:00" + \-\-change-older-than 2weeks + \-\-older @1704067200 +.TP +.BI "-o, \-\-owner " [user][:group] +Filter files by their user and/or group. Format: [(user|uid)][:(group|gid)]. Either side +is optional. Precede either side with a '!' to exclude files instead. + +Examples: + \-\-owner john + \-\-owner :students + \-\-owner "!john:students" +.TP +.BI "-C, \-\-base\-directory " path +Change the current working directory of fd to the provided path. This means that search results will +be shown with respect to the given base path. Note that relative paths which are passed to fd via the +positional \fIpath\fR argument or the \fB\-\-search\-path\fR option will also be resolved relative to +this directory. +.TP +.BI "\-\-path\-separator " separator +Set the path separator to use when printing file paths. The default is the OS-specific separator +('/' on Unix, '\\' on Windows). +.TP +.BI "\-\-search\-path " search\-path +Provide paths to search as an alternative to the positional \fIpath\fR argument. Changes the usage to +\'fd [FLAGS/OPTIONS] \-\-search\-path PATH \-\-search\-path PATH2 [PATTERN]\' +.TP +.BI "\-\-format " fmt +Specify a template string that is used for printing a line for each file found. + +The following placeholders are substituted into the string for each file before printing: +.RS +.IP {} +path (of the current search result) +.IP {/} +basename +.IP {//} +parent directory +.IP {.} +path without file extension +.IP {/.} +basename without file extension +.IP {{ +literal '{' (an escape sequence) +.IP }} +literal '}' (an escape sequence) +.P +Notice that you can use "{{" and "}}" to escape "{" and "}" respectively, which is especially +useful if you need to include the literal text of one of the above placeholders. +.RE +.TP +.BI "\-x, \-\-exec " command +.RS +Execute +.I command +for each search result in parallel (use --threads=1 for sequential command execution). + +Note that all subsequent positional arguments are considered to be arguments to the +.I command +- not to fd. +It is therefore recommended to place the \-x/\-\-exec option last. Alternatively, you can supply +a ';' argument to end the argument list and continue with more fd options. +Most shells require ';' to be escaped: '\\;'. +Specify this option multiple times to run several commands for each file or directory found. +For each file or directory, fd runs these commands one after another in the order they appear +on the command line. In that case, you must supply a ';' argument for all but the last command. +When running in parallel, fd buffers command output and prints it together for each search result, +without interleaving it with command output for other results. + +The order in which different search results are processed and their output is printed is not +guaranteed, even with --threads=1. + +Before executing the command, any placeholder patterns in the command are replaced with the +corresponding values for the current file. The same placeholders are used as in the "\-\-format" +option. + +If no placeholder is present, an implicit "{}" at the end is assumed. + +If --print0 is also given, then a null character (\\0) will be printed between the output for each found entry. +This allows another program to easily distinguish the output for each file if the command(s) produce multiple lines. + +Examples: + + - find all *.zip files and unzip them: + + fd -e zip -x unzip + + - find *.h and *.cpp files and run "clang-format -i .." for each of them: + + fd -e h -e cpp -x clang-format -i + + - Convert all *.jpg files to *.png files: + + fd -e jpg -x convert {} {.}.png + + - Run stat for each *.txt file, separated by null characters + + fd -0 -e txt -x stat +.RE +.TP +.BI "\-X, \-\-exec-batch " command +.RS +Execute +.I command +once, with all search results as arguments. + +The order of the arguments is non-deterministic and should not be relied upon. + +This uses the same placeholders as "\-\-format" and "\-\-exec", but instead of expanding +once per command invocation each argument containing a placeholder is expanding for every +file in a batch and passed as separate arguments. + +If no placeholder is present, an implicit "{}" at the end is assumed. + +Like \-\-exec, subsequent arguments are assumed to be part of +.I command +until either the end of command arguments or a ';' argument. See above. + +Like \-\-exec, this can be used multiple times, in which case each command will be run in +the order given. + +Examples: + + - Find all test_*.py files and open them in your favorite editor: + + fd -g 'test_*.py' -X vim + + Note that this executes a single "vim" process with all search results as arguments. + + - Find all *.rs files and count the lines with "wc -l ...": + + fd -e rs -X wc -l +.RE +.TP +.BI "\-\-batch-size " size +Maximum number of arguments to pass to the command given with -X. If the number of results is +greater than the given size, the command given with -X is run again with remaining arguments. A +batch size of zero means there is no limit (default), but note that batching might still happen +due to OS restrictions on the maximum length of command lines. +.SH PATTERN SYNTAX +The regular expression syntax used by fd is documented here: + + https://docs.rs/regex/1.0.0/regex/#syntax + +The glob syntax is documented here: + + https://docs.rs/globset/#syntax +.SH ENVIRONMENT +.TP +.B LS_COLORS +Determines how to colorize search results, see +.BR dircolors (1) . +.TP +.B NO_COLOR +Disables colorized output. +.TP +.B XDG_CONFIG_HOME, HOME +Used to locate the global ignore file. If +.B XDG_CONFIG_HOME +is set, use +.IR $XDG_CONFIG_HOME/fd/ignore . +Otherwise, use +.IR $HOME/.config/fd/ignore . +.SH FILES +.TP +.B .fdignore +This file works similarly to a .gitignore file anywhere in the searched tree and specifies patterns +that should be excluded from the search. However, this file is specific to fd, and will be used even +if the --no-ignore-vcs option is used. +.TP +.B $XDG_CONFIG_HOME/fd/ignore +Global ignore file. Unless ignore mode is turned off (such as with --no-ignore) +ignore entries in this file will be ignored, as if it was an .fdignore file in the +current directory. +.SH EXAMPLES +.TP +.RI "Find files and directories that match the pattern '" needle "':" +$ fd needle +.TP +.RI "Start a search in a given directory (" /var/log "):" +$ fd nginx /var/log +.TP +.RI "Find all Python files (all files with the extension " .py ") in the current directory:" +$ fd -e py +.TP +.RI "Open all search results with vim:" +$ fd pattern -X vim +.SH Tips and Tricks +.IP \[bu] +If you add ".git/" to your global ignore file ($XDG_CONFIG_HOME/fd/ignore), then +".git" folders will be ignored by default, even when the --hidden option is used. +.IP \[bu] +You can use a shell alias or a wrapper script in order to pass desired flags to fd +by default. For example if you do not like the default behavior of respecting gitignore, +you can use +`alias fd="/usr/bin/fd --no-ignore-vcs"` +in your .bashrc to create an alias for fd that doesn't ignore git files by default. +.SH BUGS +Bugs can be reported on GitHub: https://github.com/sharkdp/fd/issues +.SH SEE ALSO +.BR find (1) diff --git a/userland/fd/doc/logo.png b/userland/fd/doc/logo.png new file mode 100644 index 0000000..483dfda Binary files /dev/null and b/userland/fd/doc/logo.png differ diff --git a/userland/fd/doc/logo.svg b/userland/fd/doc/logo.svg new file mode 100644 index 0000000..780beb0 --- /dev/null +++ b/userland/fd/doc/logo.svg @@ -0,0 +1,161 @@ + + + + + + + + + + + + fd + fd + + + + + + + + + + + + + + + + + + + + + + diff --git a/userland/fd/doc/release-checklist.md b/userland/fd/doc/release-checklist.md new file mode 100644 index 0000000..5edc664 --- /dev/null +++ b/userland/fd/doc/release-checklist.md @@ -0,0 +1,69 @@ +# Release checklist + +This file can be used as-is, or copied into the GitHub PR description which includes +necessary changes for the upcoming release. + +## Version bump + +This section can be done by running `scripts/version-bump.sh` with the new version +as an argument. + +- [ ] Create a new branch for the required changes for this release. +- [ ] Update version in `Cargo.toml`. Run `cargo build` to update `Cargo.lock`. + Make sure to `git add` the `Cargo.lock` changes as well. +- [ ] Find the current min. supported Rust version by running + `grep rust-version Cargo.toml`. +- [ ] Update the `fd` version and the min. supported Rust version in `README.md`. +- [ ] Update `CHANGELOG.md`. Change the heading of the *"Upcoming release"* section + to the version of this release. + +## Pre-release checks and updates + +- [ ] Install the latest version (`cargo install --locked -f --path .`) and make + sure that it is available on the `PATH` (`fd --version` should show the + new version). +- [ ] Review `-h`, `--help`, and the `man` page. +- [ ] Run `fd -h` and copy the output to the *"Command-line options"* section in + the README. This can be done by running `gawk -i inplace -f scripts/update-help.awk README.md`. +- [ ] Push all changes and wait for CI to succeed (before continuing with the + next section). +- [ ] Optional: manually test the new features and command-line options described + in the `CHANGELOG.md`. +- [ ] Run `cargo publish --dry-run` to make sure that it will succeed later + (after creating the GitHub release). + +## Release + +- [ ] Merge your release branch (should be a fast-forward merge). +- [ ] Create a tag and push it: `git tag vX.Y.Z; git push origin tag vX.Y.Z`. + This will trigger the deployment via GitHub Actions. + REMINDER: If your `origin` is a fork, don't forget to push to e.g. `upstream` + instead. +- [ ] Go to https://github.com/sharkdp/fd/releases/new to create the new + release. Select the new tag and also use it as the release title. For the + release notes, copy the corresponding section from `CHANGELOG.md` and + possibly add additional remarks for package maintainers. + Publish the release. +- [ ] Check if the binary deployment works (archives and Debian packages should + appear when the CI run *for the Git tag* has finished). +- [ ] Publish to crates.io by running `cargo publish` in a *clean* repository. + One way to do this is to clone a fresh copy. + +## Post-release + +- [ ] Prepare a new *"Upcoming release"* section at the top of `CHANGELOG.md`. + Put this at the top: + + # Upcoming release + + ## Features + + + ## Bugfixes + + + ## Changes + + + ## Other + diff --git a/userland/fd/doc/screencast.sh b/userland/fd/doc/screencast.sh new file mode 100644 index 0000000..cb9d551 --- /dev/null +++ b/userland/fd/doc/screencast.sh @@ -0,0 +1,64 @@ +#!/bin/bash +# Designed to be executed via svg-term from the fd root directory: +# svg-term --command="bash doc/screencast.sh" --out doc/screencast.svg --padding=10 +# Then run this (workaround for #1003): +# sed -i '' 's/▶▶fâ–¶fdbuild.rsclippy.tomlcontrib/contrib/completion/contrib/completion/_fddoc/doc/fd.1doc/logo.pngdoc/logo.svgdoc/screencast.shdoc/screencast.svgsrc/src/app.rssrc/config.rssrc/exec/src/exec/mod.rssrc/exec/token.rssrc/exit_codes.rssrc/filesystem.rssrc/filetypes.rssrc/filter/src/filter/mod.rssrc/filter/owner.rssrc/filter/size.rssrc/filter/time.rssrc/main.rssrc/output.rssrc/regex_helper.rssrc/walk.rstests/tests/testenv/tests/testenv/mod.rstests/tests.rsâ–¶fdappâ–¶fdfâ–¶fdfiâ–¶fdfi--typeâ–¶fdfi--typefâ–¶fd-â–¶fd--typeâ–¶fd--typedâ–¶fd-eâ–¶fd-emâ–¶fd-emdCHANGELOG.mdCONTRIBUTING.mdREADME.mdâ–¶fd-emd--execâ–¶fd-emd--execwcâ–¶fd-emd--execwc-l707./README.md37./CONTRIBUTING.md606./CHANGELOG.mdâ–¶fd'^[A-Z]'Cargo.lockCargo.tomlLICENSE-APACHELICENSE-MITâ–¶fd--â–¶fd--excludeâ–¶fd--excludesrcâ–¶fd--hiddenâ–¶fd--hiddensample.git/hooks/applypatch-msg.sample.git/hooks/commit-msg.sample.git/hooks/fsmonitor-watchman.sample.git/hooks/post-update.sample.git/hooks/pre-applypatch.sample.git/hooks/pre-commit.sample.git/hooks/pre-merge-commit.sample.git/hooks/pre-push.sample.git/hooks/pre-rebase.sample.git/hooks/pre-receive.sample.git/hooks/prepare-commit-msg.sample.git/hooks/push-to-checkout.sample.git/hooks/update.samplesrc/dir_entry.rssrc/error.rssrc/exec/command.rssrc/exec/input.rssrc/exec/job.rsâ–¶fdaâ–¶fdapâ–¶fdfi-â–¶fdfi--â–¶fdfi--tâ–¶fdfi--tyâ–¶fdfi--typâ–¶fd--tâ–¶fd--tyâ–¶fd--typâ–¶fd-emd-â–¶fd-emd--â–¶fd-emd--eâ–¶fd-emd--exâ–¶fd-emd--exeâ–¶fd-emd--execwâ–¶fd-emd--execwc-â–¶fd'â–¶fd'^[â–¶fd'^[Aâ–¶fd'^[A-â–¶fd'^[A-Zâ–¶fd'^[A-Z]â–¶fd--eâ–¶fd--exâ–¶fd--excâ–¶fd--exclâ–¶fd--excluâ–¶fd--excludâ–¶fd--excludesâ–¶fd--excludesrâ–¶fd--hâ–¶fd--hiâ–¶fd--hidâ–¶fd--hiddâ–¶fd--hiddeâ–¶fd--hiddensâ–¶fd--hiddensaâ–¶fd--hiddensamâ–¶fd--hiddensampâ–¶fd--hiddensampl \ No newline at end of file diff --git a/userland/fd/doc/sponsors.md b/userland/fd/doc/sponsors.md new file mode 100644 index 0000000..141ff74 --- /dev/null +++ b/userland/fd/doc/sponsors.md @@ -0,0 +1,12 @@ +## Sponsors + +`fd` development is sponsored by many individuals and companies. Thank you very much! + +Please note, that being sponsored does not affect the individuality of the `fd` +project or affect the maintainers' actions in any way. +We remain impartial and continue to assess pull requests solely on merit - the +features added, bugs solved, and effect on the overall complexity of the code. +No issue will have a different priority based on sponsorship status of the +reporter. + +Contributions from anybody are most welcomed, please see our [`CONTRIBUTING.md`](../CONTRIBUTING.md) guide. diff --git a/userland/fd/rustfmt.toml b/userland/fd/rustfmt.toml new file mode 100644 index 0000000..a935f8f --- /dev/null +++ b/userland/fd/rustfmt.toml @@ -0,0 +1 @@ +# Defaults are used diff --git a/userland/fd/scripts/create-archive.sh b/userland/fd/scripts/create-archive.sh new file mode 100755 index 0000000..ad0823a --- /dev/null +++ b/userland/fd/scripts/create-archive.sh @@ -0,0 +1,67 @@ +#!/bin/bash +# +# Create a .tar.gz archive of the fd project for distribution +# +# Expected environment variables: +# +# TARGET: rust target to publish. Defaults to default rustc target +# BIN_PATH: path to fd binary +# PKG_STAGING: directory to use for staging files to include in the archive +# ARCHIVE_NAME: Name of output file, minus extension, defaults to "fd" with a version appended +# if VERSION is specified +# VERSION: version to publish +# GITHUB_OUTPUT: file to store github output variables in + +if [[ -z "$TARGET" ]]; then + TARGET="$(rustc -vV | sed -n 's/host: //p')" +fi + +PKG_suffix=".tar.gz" +EXE_suffix="" +case ${TARGET} in +*-pc-windows-*) + PKG_suffix=".zip" + EXE_suffix=".exe" + ;; +esac + +if [[ -z $BIN_PATH ]]; then + BIN_PATH=target/${PROFILE:-release}/fd +fi + +if [[ -z "$ARCHIVE_NAME" ]]; then + ARCHIVE_NAME=fd + [[ -n "$VERSION" ]] && ARCHIVE_NAME+="-v$VERSION" +fi + +PKG_NAME=${ARCHIVE_NAME}${PKG_suffix} + +staging_dir="${PKG_STAGING:-package}" +ARCHIVE_DIR="${staging_dir}/${ARCHIVE_NAME}/" +mkdir -p "${ARCHIVE_DIR}" + +# Binary +cp "${BIN_PATH}" "$ARCHIVE_DIR" + +# README, LICENSE and CHANGELOG files +cp "README.md" "LICENSE-MIT" "LICENSE-APACHE" "CHANGELOG.md" "$ARCHIVE_DIR" + +# Man page +cp "doc/fd.1" "$ARCHIVE_DIR" + +# Autocompletion files +cp -r autocomplete "${ARCHIVE_DIR}" + +# base compressed package +pushd "${staging_dir}/" >/dev/null +case ${TARGET} in +*-pc-windows-*) 7z -y a "${PKG_NAME}" "${ARCHIVE_NAME}"/* | tail -2 ;; +*) tar czf "${PKG_NAME}" "${ARCHIVE_NAME}"/* ;; +esac; +popd >/dev/null + +if [[ -n "$GITHUB_OUTPUT" ]]; then + echo "PKG_NAME=${PKG_NAME}" >> $GITHUB_OUTPUT + # Let subsequent steps know where to find the compressed package + echo "PKG_PATH=${PKG_STAGING}/${PKG_NAME}" >> $GITHUB_OUTPUT +fi diff --git a/userland/fd/scripts/create-deb.sh b/userland/fd/scripts/create-deb.sh new file mode 100755 index 0000000..1f1767f --- /dev/null +++ b/userland/fd/scripts/create-deb.sh @@ -0,0 +1,135 @@ +#!/bin/bash +COPYRIGHT_YEARS="2018 - "$(date "+%Y") +MAINTAINER="David Peter " +REPO="https://github.com/sharkdp/fd" +DPKG_STAGING="${CICD_INTERMEDIATES_DIR:-.}/debian-package" +DPKG_DIR="${DPKG_STAGING}/dpkg" +mkdir -p "${DPKG_DIR}" + +if [[ -z "$TARGET" ]]; then + TARGET="$(rustc -vV | sed -n 's|host: \(.*\)|\1|p')" +fi + +case "$TARGET" in + *-musl*) + DPKG_BASENAME=fd-musl + DPKG_CONFLICTS="fd, fd-find" + ;; + *) + DPKG_BASENAME=fd + DPKG_CONFLICTS="fd-musl, fd-find" + ;; +esac + +if [[ -z "$DPKG_VERSION" ]]; then + DPKG_VERSION=$(cargo metadata --no-deps --format-version 1 | jq -r .packages[0].version) +fi + +unset DPKG_ARCH +case "${TARGET}" in + aarch64-*-linux-*) DPKG_ARCH=arm64 ;; + arm-*-linux-*hf) DPKG_ARCH=armhf ;; + i686-*-linux-*) DPKG_ARCH=i686 ;; + x86_64-*-linux-*) DPKG_ARCH=amd64 ;; + *) DPKG_ARCH=notset ;; +esac; + +DPKG_NAME="${DPKG_BASENAME}_${DPKG_VERSION}_${DPKG_ARCH}.deb" + +BIN_PATH=${BIN_PATH:-target/${TARGET}/release/fd} + +# Binary +install -Dm755 "${BIN_PATH}" "${DPKG_DIR}/usr/bin/fd" + +# Man page +install -Dm644 'doc/fd.1' "${DPKG_DIR}/usr/share/man/man1/fd.1" +gzip -n --best "${DPKG_DIR}/usr/share/man/man1/fd.1" + +# Autocompletion files +install -Dm644 'autocomplete/fd.bash' "${DPKG_DIR}/usr/share/bash-completion/completions/fd" +install -Dm644 'autocomplete/fd.fish' "${DPKG_DIR}/usr/share/fish/vendor_completions.d/fd.fish" +install -Dm644 'autocomplete/_fd' "${DPKG_DIR}/usr/share/zsh/vendor-completions/_fd" + +# README and LICENSE +install -Dm644 "README.md" "${DPKG_DIR}/usr/share/doc/${DPKG_BASENAME}/README.md" +install -Dm644 "LICENSE-MIT" "${DPKG_DIR}/usr/share/doc/${DPKG_BASENAME}/LICENSE-MIT" +install -Dm644 "LICENSE-APACHE" "${DPKG_DIR}/usr/share/doc/${DPKG_BASENAME}/LICENSE-APACHE" +install -Dm644 "CHANGELOG.md" "${DPKG_DIR}/usr/share/doc/${DPKG_BASENAME}/changelog" +gzip -n --best "${DPKG_DIR}/usr/share/doc/${DPKG_BASENAME}/changelog" + +# Create symlink and completions so fdfind can be used as well: +ln -s "/usr/bin/fd" "${DPKG_DIR}/usr/bin/fdfind" + +install -Dm644 'autocomplete/fdfind.bash' "${DPKG_DIR}/usr/share/bash-completion/completions/fdfind" +install -Dm644 'autocomplete/fdfind.fish' "${DPKG_DIR}/usr/share/fish/vendor_completions.d/fdfind.fish" +install -Dm644 'autocomplete/_fdfind' "${DPKG_DIR}/usr/share/zsh/vendor-completions/_fdfind" + +cat > "${DPKG_DIR}/usr/share/doc/${DPKG_BASENAME}/copyright" < "${DPKG_DIR}/DEBIAN/control" <> "$GITHUB_OUTPUT" + echo "DPKG_PATH=${DPKG_PATH}" >> "$GITHUB_OUTPUT" +fi + +# build dpkg +fakeroot dpkg-deb --build "${DPKG_DIR}" "${DPKG_PATH}" diff --git a/userland/fd/scripts/update-help.awk b/userland/fd/scripts/update-help.awk new file mode 100755 index 0000000..269317d --- /dev/null +++ b/userland/fd/scripts/update-help.awk @@ -0,0 +1,41 @@ +#!/usr/bin/gawk -f + +BEGIN { + inSection = 0 + inBlock = 0 +} + + +/Command-line options/ { inSection=1 } + +inSection && /^```/ { + inBlock=1 + inSection=0 + # print the starting fence then move to next line + print + next +} + +inBlock && /^```/ { + cmd="cargo run --release --quiet -- -h" + # Output the results of fd -h + u=0 + while (cmd | getline line) { + # Skip everything before the usage line + if (line ~ /^Usage/) { + u=1; + } + if (u) { + print line + } + } + status = close(cmd) + if (status) { + print "failed to generate help output" > "/dev/stderr" + exit 1 + } + inBlock = 0 +} + +!inBlock + diff --git a/userland/fd/scripts/version-bump.sh b/userland/fd/scripts/version-bump.sh new file mode 100755 index 0000000..dd2f5e3 --- /dev/null +++ b/userland/fd/scripts/version-bump.sh @@ -0,0 +1,22 @@ +#!/usr/bin/bash + +set -eu + +# This script automates the "Version bump" section + +version="$1" + +if [[ -z $version ]]; then + echo "Usage: must supply version as first argument" >&2 + exit 1 +fi + +git switch -C "release-$version" +sed -i -e "0,/^\[badges/{s/^version =.*/version = \"$version\"/}" Cargo.toml + +msrv="$(grep -F rust-version Cargo.toml | sed -e 's/^rust-version= "\(.*\)"/\1/')" + +sed -i -e "s/Note that rust version \*[0-9.]+\* or later/Note that rust version *$msrv* or later/" README.md + +sed -i -e "s/^# Upcoming release/# $version/" CHANGELOG.md + diff --git a/userland/fd/src/cli.rs b/userland/fd/src/cli.rs new file mode 100644 index 0000000..4e6e3b6 --- /dev/null +++ b/userland/fd/src/cli.rs @@ -0,0 +1,983 @@ +use std::num::NonZeroUsize; +use std::path::{Path, PathBuf}; +use std::time::Duration; + +use anyhow::anyhow; +use clap::{ + Arg, ArgAction, ArgGroup, ArgMatches, Command, Parser, ValueEnum, builder::ArgPredicate, + error::ErrorKind, value_parser, +}; +#[cfg(feature = "completions")] +use clap_complete::Shell; +use normpath::PathExt; + +use crate::exec::CommandSet; +use crate::filesystem; +#[cfg(unix)] +use crate::filter::OwnerFilter; +use crate::filter::SizeFilter; + +#[derive(Parser)] +#[command( + name = "fd", + version, + about = "A program to find entries in your filesystem with regex and glob based matching. By default, fd respects gitignore rules, ignores hidden directories, and is case insensitive.", + after_long_help = "Bugs can be reported on GitHub: https://github.com/sharkdp/fd/issues", + max_term_width = 98, + args_override_self = true, + group(ArgGroup::new("execs").args(&["exec", "exec_batch", "list_details"]).conflicts_with_all(&[ + "max_results", "quiet", "max_one_result"])), +)] +pub struct Opts { + /// Include hidden directories and files in the search results (default: + /// hidden files and directories are skipped). Files and directories are + /// considered to be hidden if their name starts with a `.` sign (dot). + /// Any files or directories that are ignored due to the rules described by + /// --no-ignore are still ignored unless otherwise specified. + /// The flag can be overridden with --no-hidden. + #[arg( + long, + short = 'H', + help = "Search hidden files and directories", + long_help, + default_value_ifs([ + ("no_hidden", ArgPredicate::IsPresent, Some("false")), + ("rg_alias_hidden_ignore", ArgPredicate::IsPresent, Some("true")), + ]) + )] + pub hidden: bool, + + /// Overrides --hidden + #[arg(long, overrides_with = "hidden", hide = true, action = ArgAction::SetTrue)] + no_hidden: (), + + /// Show search results from files and directories that would otherwise be + /// ignored by '.gitignore', '.ignore', '.fdignore', or the global ignore file. + /// The flag can be overridden with --ignore. + #[arg( + long, + short = 'I', + help = "Do not respect .(git|fd)ignore files", + long_help, + default_value_ifs([ + ("ignore", ArgPredicate::IsPresent, Some("false")), + ("rg_alias_hidden_ignore", ArgPredicate::IsPresent, Some("true")), + ]) + )] + pub no_ignore: bool, + + /// Overrides --no-ignore + #[arg(long, overrides_with = "no_ignore", hide = true, action = ArgAction::SetTrue)] + ignore: (), + + ///Show search results from files and directories that + ///would otherwise be ignored by '.gitignore' files. + ///The flag can be overridden with --ignore-vcs. + #[arg( + long, + hide_short_help = true, + help = "Do not respect .gitignore files", + long_help + )] + pub no_ignore_vcs: bool, + + /// Overrides --no-ignore-vcs + #[arg(long, overrides_with = "no_ignore_vcs", hide = true, action = ArgAction::SetTrue)] + ignore_vcs: (), + + /// Do not require a git repository to respect gitignores. + /// By default, fd will only respect global gitignore rules, .gitignore rules, + /// and local exclude rules if fd detects that you are searching inside a + /// git repository. This flag allows you to relax this restriction such that + /// fd will respect all git related ignore rules regardless of whether you're + /// searching in a git repository or not. + /// + /// + /// This flag can be disabled with --require-git. + #[arg( + long, + overrides_with = "require_git", + hide_short_help = true, + // same description as ripgrep's flag: ripgrep/crates/core/app.rs + long_help + )] + pub no_require_git: bool, + + /// Overrides --no-require-git + #[arg(long, overrides_with = "no_require_git", hide = true, action = ArgAction::SetTrue)] + require_git: (), + + /// Show search results from files and directories that would otherwise be + /// ignored by '.gitignore', '.ignore', or '.fdignore' files in parent directories. + /// The flag can be overridden with --ignore-parent. + #[arg( + long, + hide_short_help = true, + help = "Do not respect .(git|fd)ignore files in parent directories", + long_help + )] + pub no_ignore_parent: bool, + + /// Overrides --no-ignore-parent + #[arg(long, overrides_with = "no_ignore_parent", hide = true, action = ArgAction::SetTrue)] + ignore_parent: (), + + /// Do not respect the global ignore file + #[arg(long, hide = true)] + pub no_global_ignore_file: bool, + + /// Perform an unrestricted search, including ignored and hidden files. This is + /// an alias for '--no-ignore --hidden'. + #[arg( + long = "unrestricted", + short = 'u', + hide_short_help = true, + help = "Unrestricted search, alias for '--no-ignore --hidden'", + long_help + )] + rg_alias_hidden_ignore: bool, + + /// Case-sensitive search (default: smart case) + #[arg( + long, + short = 's', + overrides_with("ignore_case"), + long_help = "Perform a case-sensitive search. By default, fd uses case-insensitive \ + searches, unless the pattern contains an uppercase character (smart \ + case)." + )] + pub case_sensitive: bool, + + /// Perform a case-insensitive search. By default, fd uses case-insensitive + /// searches, unless the pattern contains an uppercase character (smart + /// case). + #[arg( + long, + short = 'i', + overrides_with("case_sensitive"), + help = "Case-insensitive search (default: smart case)", + long_help + )] + pub ignore_case: bool, + + /// Perform a glob-based search instead of a regular expression search. + #[arg( + long, + short = 'g', + conflicts_with("fixed_strings"), + help = "Glob-based search (default: regular expression)", + long_help + )] + pub glob: bool, + + /// Perform a regular-expression based search (default). This can be used to + /// override --glob. + #[arg( + long, + overrides_with("glob"), + hide_short_help = true, + help = "Regular-expression based search (default)", + long_help + )] + pub regex: bool, + + /// Treat the pattern as a literal string instead of a regular expression. Note + /// that this also performs substring comparison. If you want to match on an + /// exact filename, consider using '--glob' or '--exact' instead. + #[arg( + long, + short = 'F', + alias = "literal", + hide_short_help = true, + help = "Treat pattern as literal string instead of regex", + long_help + )] + pub fixed_strings: bool, + + /// Perform an exact match. This is equivalent to '--fixed-strings' but requires + /// the pattern to match the entire filename (or path if '--full-path' is used), + /// rather than a substring. Special regex characters in the pattern are treated + /// as literal characters. + #[arg( + long, + conflicts_with_all(["glob", "fixed_strings"]), + hide_short_help = true, + help = "Match the entire filename exactly (literal, non-substring)", + long_help + )] + pub exact: bool, + + /// Add additional required search patterns, all of which must be matched. Multiple + /// additional patterns can be specified. The patterns are regular + /// expressions, unless '--glob' or '--fixed-strings' is used. + #[arg( + long = "and", + value_name = "pattern", + help = "Additional search patterns that need to be matched", + long_help, + hide_short_help = true, + allow_hyphen_values = true + )] + pub exprs: Option>, + + /// Shows the full path starting from the root as opposed to relative paths. + /// The flag can be overridden with --relative-path. + #[arg( + long, + short = 'a', + help = "Show absolute instead of relative paths", + long_help + )] + pub absolute_path: bool, + + /// Overrides --absolute-path + #[arg(long, overrides_with = "absolute_path", hide = true, action = ArgAction::SetTrue)] + relative_path: (), + + /// Use a detailed listing format like 'ls -l'. This is basically an alias + /// for '--exec-batch ls -l' with some additional 'ls' options. This can be + /// used to see more metadata, to show symlink targets and to achieve a + /// deterministic sort order. + #[arg( + long, + short = 'l', + conflicts_with("absolute_path"), + help = "Use a long listing format with file metadata", + long_help + )] + pub list_details: bool, + + /// Follow symbolic links + #[arg( + long, + short = 'L', + alias = "dereference", + long_help = "By default, fd does not descend into symlinked directories. Using this \ + flag, symbolic links are also traversed. \ + Flag can be overridden with --no-follow." + )] + pub follow: bool, + + /// Overrides --follow + #[arg(long, overrides_with = "follow", hide = true, action = ArgAction::SetTrue)] + no_follow: (), + + /// By default, the search pattern is only matched against the filename (or directory name). Using this flag, the pattern is matched against the full (absolute) path. Example: + /// fd --glob -p '**/.git/config' + #[arg( + long, + short = 'p', + help = "Search full abs. path (default: filename only)", + long_help, + verbatim_doc_comment + )] + pub full_path: bool, + + /// Separate search results by the null character (instead of newlines). + /// Useful for piping results to 'xargs'. + #[arg( + long = "print0", + short = '0', + conflicts_with("list_details"), + hide_short_help = true, + help = "Separate search results by the null character", + long_help + )] + pub null_separator: bool, + + /// Limit the directory traversal to a given depth. By default, there is no + /// limit on the search depth. + #[arg( + long, + short = 'd', + value_name = "depth", + alias("maxdepth"), + help = "Set maximum search depth (default: none)", + long_help + )] + max_depth: Option, + + /// Only show search results starting at the given depth. + /// See also: '--max-depth' and '--exact-depth' + #[arg( + long, + value_name = "depth", + hide_short_help = true, + alias("mindepth"), + help = "Only show search results starting at the given depth.", + long_help + )] + min_depth: Option, + + /// Only show search results at the exact given depth. This is an alias for + /// '--min-depth --max-depth '. + #[arg(long, value_name = "depth", hide_short_help = true, conflicts_with_all(&["max_depth", "min_depth"]), + help = "Only show search results at the exact given depth", + long_help, + )] + exact_depth: Option, + + /// Exclude files/directories that match the given glob pattern. This + /// overrides any other ignore logic. Multiple exclude patterns can be + /// specified. + /// + /// Examples: + /// {n} --exclude '*.pyc' + /// {n} --exclude node_modules + #[arg( + long, + short = 'E', + value_name = "glob", + help = "Exclude entries that match the given glob pattern", + long_help + )] + pub exclude: Vec, + + /// Do not traverse into directories that match the search criteria. If + /// you want to exclude specific directories, use the '--exclude=…' option. + #[arg(long, hide_short_help = true, conflicts_with_all(&["size", "exact_depth"]), + long_help, + )] + pub prune: bool, + + /// Filter the search by type: + /// {n} 'f' or 'file': regular files + /// {n} 'd' or 'dir' or 'directory': directories + /// {n} 'l' or 'symlink': symbolic links + /// {n} 's' or 'socket': socket + /// {n} 'p' or 'pipe': named pipe (FIFO) + /// {n} 'b' or 'block-device': block device + /// {n} 'c' or 'char-device': character device + /// {n}{n} 'x' or 'executable': executables + /// {n} 'e' or 'empty': empty files or directories + /// + /// This option can be specified more than once to include multiple file types. + /// Searching for '--type file --type symlink' will show both regular files as + /// well as symlinks. Note that the 'executable' and 'empty' filters work differently: + /// '--type executable' implies '--type file' by default. And '--type empty' searches + /// for empty files and directories, unless either '--type file' or '--type directory' + /// is specified in addition. + /// + /// Examples: + /// {n} - Only search for files: + /// {n} fd --type file … + /// {n} fd -tf … + /// {n} - Find both files and symlinks + /// {n} fd --type file --type symlink … + /// {n} fd -tf -tl … + /// {n} - Find executable files: + /// {n} fd --type executable + /// {n} fd -tx + /// {n} - Find empty files: + /// {n} fd --type empty --type file + /// {n} fd -te -tf + /// {n} - Find empty directories: + /// {n} fd --type empty --type directory + /// {n} fd -te -td + #[arg( + long = "type", + short = 't', + value_name = "filetype", + hide_possible_values = true, + value_enum, + help = "Filter by type: file (f), directory (d/dir), symlink (l), \ + executable (x), empty (e), socket (s), pipe (p), \ + char-device (c), block-device (b)", + long_help + )] + pub filetype: Option>, + + /// Filter results by extension. By default, this matches all entry types + /// (including directories whose names end with the extension). Use `--type` + /// to restrict the search to specific types. Multiple extensions can be specified. + /// + /// If you want to search for files without extension, + /// you can use the regex '^[^.]+$' as a normal search pattern. + #[arg( + long = "extension", + short = 'e', + value_name = "ext", + help = "Filter by extension", + long_help + )] + pub extensions: Option>, + + /// Limit results based on the size of files using the format <+->. + /// '+': file size must be greater than or equal to this + /// '-': file size must be less than or equal to this + /// + /// If neither '+' nor '-' is specified, file size must be exactly equal to this. + /// 'NUM': The numeric size (e.g. 500) + /// 'UNIT': The units for NUM. They are not case-sensitive. + /// Allowed unit values: + /// 'b': bytes + /// 'k': kilobytes (base ten, 10^3 = 1000 bytes) + /// 'm': megabytes + /// 'g': gigabytes + /// 't': terabytes + /// 'ki': kibibytes (base two, 2^10 = 1024 bytes) + /// 'mi': mebibytes + /// 'gi': gibibytes + /// 'ti': tebibytes + #[arg(long, short = 'S', value_parser = SizeFilter::from_string, allow_hyphen_values = true, verbatim_doc_comment, value_name = "size", + help = "Limit results based on the size of files", + long_help, + verbatim_doc_comment, + )] + pub size: Vec, + + /// Filter results based on the file modification time. Files with modification times + /// greater than the argument are returned. The argument can be provided + /// as a specific point in time (YYYY-MM-DD HH:MM:SS or @timestamp) or as a duration (10h, 1d, 35min). + /// If the time is not specified, it defaults to 00:00:00. + /// '--change-newer-than', '--newer', or '--changed-after' can be used as aliases. + /// + /// Examples: + /// {n} --changed-within 2weeks + /// {n} --change-newer-than '2018-10-27 10:00:00' + /// {n} --newer 2018-10-27 + /// {n} --changed-after 1day + #[arg( + long, + alias("change-newer-than"), + alias("newer"), + alias("changed-after"), + value_name = "date|dur", + help = "Filter by file modification time (newer than)", + long_help + )] + pub changed_within: Option, + + /// Filter results based on the file modification time. Files with modification times + /// less than the argument are returned. The argument can be provided + /// as a specific point in time (YYYY-MM-DD HH:MM:SS or @timestamp) or as a duration (10h, 1d, 35min). + /// '--change-older-than' or '--older' can be used as aliases. + /// + /// Examples: + /// {n} --changed-before '2018-10-27 10:00:00' + /// {n} --change-older-than 2weeks + /// {n} --older 2018-10-27 + #[arg( + long, + alias("change-older-than"), + alias("older"), + value_name = "date|dur", + help = "Filter by file modification time (older than)", + long_help + )] + pub changed_before: Option, + + /// Filter files by their user and/or group. + /// Format: [(user|uid)][:(group|gid)]. Either side is optional. + /// Precede either side with a '!' to exclude files instead. + /// + /// Examples: + /// {n} --owner john + /// {n} --owner :students + /// {n} --owner '!john:students' + #[cfg(unix)] + #[arg(long, short = 'o', value_parser = OwnerFilter::from_string, value_name = "user:group", + help = "Filter by owning user and/or group", + long_help, + )] + pub owner: Option, + + /// Instead of printing the file normally, print the format string with the following placeholders replaced: + /// '{}': path (of the current search result) + /// '{/}': basename + /// '{//}': parent directory + /// '{.}': path without file extension + /// '{/.}': basename without file extension + #[arg( + long, + value_name = "fmt", + help = "Print results according to template", + conflicts_with = "list_details", + allow_hyphen_values = true + )] + pub format: Option, + + #[command(flatten)] + pub exec: Exec, + + /// Maximum number of arguments to pass to the command given with -X. + /// If the number of results is greater than the given size, + /// the command given with -X is run again with remaining arguments. + /// A batch size of zero means there is no limit (default), but note + /// that batching might still happen due to OS restrictions on the + /// maximum length of command lines. + #[arg( + long, + value_name = "size", + hide_short_help = true, + requires("exec_batch"), + value_parser = value_parser!(usize), + default_value_t, + help = "Max number of arguments to run as a batch size with -X", + long_help, + )] + pub batch_size: usize, + + /// Add a custom ignore-file in '.gitignore' format. These files have a low precedence. + #[arg( + long, + value_name = "path", + hide_short_help = true, + help = "Add a custom ignore-file in '.gitignore' format", + long_help + )] + pub ignore_file: Vec, + + /// Declare when to use color for the pattern match output + #[arg( + long, + short = 'c', + value_enum, + default_value_t = ColorWhen::Auto, + value_name = "when", + help = "When to use colors", + long_help, + )] + pub color: ColorWhen, + + /// Add a terminal hyperlink to a file:// url for each path in the output. + /// + /// Auto mode is used if no argument is given to this option. + /// + /// This doesn't do anything for --exec and --exec-batch. + #[arg( + long, + alias = "hyper", + value_name = "when", + require_equals = true, + value_enum, + default_value_t = HyperlinkWhen::Never, + default_missing_value = "auto", + num_args = 0..=1, + help = "Add hyperlinks to output paths" + )] + pub hyperlink: HyperlinkWhen, + + /// Ignore directories containing the named entry. + #[arg(long, value_name = "name")] + pub ignore_contain: Vec, + + /// Set number of threads to use for searching & executing (default: number + /// of available CPU cores, up to a maximum of 64) + #[arg(long, short = 'j', value_name = "num", hide_short_help = true, value_parser = str::parse::)] + pub threads: Option, + + /// Milliseconds to buffer before streaming search results to console + /// + /// Amount of time in milliseconds to buffer, before streaming the search + /// results to the console. + #[arg(long, hide = true, value_parser = parse_millis)] + pub max_buffer_time: Option, + + ///Limit the number of search results to 'count' and quit immediately. + #[arg( + long, + value_name = "count", + hide_short_help = true, + overrides_with("max_one_result"), + help = "Limit the number of search results", + long_help + )] + max_results: Option, + + /// Limit the search to a single result and quit immediately. + /// This is an alias for '--max-results=1'. + #[arg( + short = '1', + hide_short_help = true, + overrides_with("max_results"), + help = "Limit search to a single result", + long_help + )] + max_one_result: bool, + + /// When the flag is present, the program does not print anything and will + /// return with an exit code of 0 if there is at least one match. Otherwise, the + /// exit code will be 1. + /// '--has-results' can be used as an alias. + #[arg( + long, + short = 'q', + alias = "has-results", + hide_short_help = true, + conflicts_with("max_results"), + help = "Print nothing, exit code 0 if match found, 1 otherwise", + long_help + )] + pub quiet: bool, + + /// Enable the display of filesystem errors for situations such as + /// insufficient permissions or dead symlinks. + #[arg( + long, + hide_short_help = true, + help = "Show filesystem errors", + long_help + )] + pub show_errors: bool, + + /// Change the current working directory of fd to the provided path. This + /// means that search results will be shown with respect to the given base + /// path. Note that relative paths which are passed to fd via the positional + /// argument or the '--search-path' option will also be resolved + /// relative to this directory. + #[arg( + long, + short = 'C', + value_name = "path", + hide_short_help = true, + help = "Change current working directory", + long_help + )] + pub base_directory: Option, + + /// the search pattern which is either a regular expression (default) or a glob + /// pattern (if --glob is used). If no pattern has been specified, every entry + /// is considered a match. If your pattern starts with a dash (-), make sure to + /// pass '--' first, or it will be considered as a flag (fd -- '-foo'). + #[arg( + value_name = "pattern", + help = "the search pattern (a regular expression, unless '--glob' is used; optional)", + long_help + )] + pub pattern: Option, + + /// Set the path separator to use when printing file paths. The default is + /// the OS-specific separator ('/' on Unix, '\' on Windows). + #[arg( + long, + value_name = "separator", + hide_short_help = true, + help = "Set path separator when printing file paths", + long_help + )] + pub path_separator: Option, + + /// The directory where the filesystem search is rooted (optional). If + /// omitted, search the current working directory. + /// If supplied, the pattern must come first. + #[arg(action = ArgAction::Append, + value_name = "path", + help = "the root directories for the filesystem search (optional, requires pattern first)", + long_help, + )] + path: Vec, + + /// Provide paths to search as an alternative to the positional + /// argument. Changes the usage to `fd [OPTIONS] --search-path + /// --search-path []` + #[arg( + long, + conflicts_with("path"), + value_name = "search-path", + hide_short_help = true, + help = "Provides paths to search as an alternative to the positional argument", + long_help + )] + search_path: Vec, + + /// By default, relative paths are prefixed with './' when -x/--exec, + /// -X/--exec-batch, or -0/--print0 are given, to reduce the risk of a + /// path starting with '-' being treated as a command line option. Use + /// this flag to change this behavior. If this flag is used without a value, + /// it is equivalent to passing "always". + #[arg(long, conflicts_with_all(&["path", "search_path"]), value_name = "when", hide_short_help = true, require_equals = true, long_help)] + strip_cwd_prefix: Option>, + + /// By default, fd will traverse the file system tree as far as other options + /// dictate. With this flag, fd ensures that it does not descend into a + /// different file system than the one it started in. Comparable to the -mount + /// or -xdev filters of find(1). + #[cfg(any(unix, windows))] + #[arg(long, aliases(&["mount", "xdev"]), hide_short_help = true, long_help)] + pub one_file_system: bool, + + #[cfg(feature = "completions")] + #[arg(long, hide = true, exclusive = true)] + gen_completions: Option>, +} + +impl Opts { + pub fn search_paths(&self) -> anyhow::Result> { + // would it make sense to concatenate these? + let paths = if !self.path.is_empty() { + &self.path + } else if !self.search_path.is_empty() { + &self.search_path + } else { + let current_directory = Path::new("./"); + ensure_current_directory_exists(current_directory)?; + return Ok(vec![self.normalize_path(current_directory)]); + }; + Ok(paths + .iter() + .filter_map(|path| { + if filesystem::is_existing_directory(path) { + Some(self.normalize_path(path)) + } else { + // We use debug for the path to make it more readable if it has special characters + // Should we use a more reliable escape? + print_error!( + "Search path {:?} is not a directory.", + path.to_string_lossy().as_ref() + ); + None + } + }) + .collect()) + } + + fn normalize_path(&self, path: &Path) -> PathBuf { + if self.absolute_path { + filesystem::absolute_path(path.normalize().unwrap().as_path()).unwrap() + } else if path == Path::new(".") { + // Change "." to "./" as a workaround for https://github.com/BurntSushi/ripgrep/pull/2711 + PathBuf::from("./") + } else if path == Path::new("-") { + // Prefix "-" so the underlying walker treats it as a path. + // See sharkdp/fd#849. + Path::new(".").join(path) + } else { + path.to_path_buf() + } + } + + pub fn no_search_paths(&self) -> bool { + self.path.is_empty() && self.search_path.is_empty() + } + + pub fn max_depth(&self) -> Option { + self.max_depth.or(self.exact_depth) + } + + pub fn min_depth(&self) -> Option { + self.min_depth.or(self.exact_depth) + } + + pub fn threads(&self) -> NonZeroUsize { + self.threads.unwrap_or_else(default_num_threads) + } + + pub fn max_results(&self) -> Option { + self.max_results + .filter(|&m| m > 0) + .or_else(|| self.max_one_result.then_some(1)) + } + + pub fn strip_cwd_prefix bool>(&self, auto_pred: P) -> bool { + use self::StripCwdWhen::*; + self.no_search_paths() + && match self.strip_cwd_prefix.map_or(Auto, |o| o.unwrap_or(Always)) { + Auto => auto_pred(), + Always => true, + Never => false, + } + } + + #[cfg(feature = "completions")] + pub fn gen_completions(&self) -> anyhow::Result> { + self.gen_completions + .map(|maybe_shell| match maybe_shell { + Some(sh) => Ok(sh), + None => { + Shell::from_env().ok_or_else(|| anyhow!("Unable to get shell from environment")) + } + }) + .transpose() + } +} + +/// Get the default number of threads to use, if not explicitly specified. +fn default_num_threads() -> NonZeroUsize { + // If we can't get the amount of parallelism for some reason, then + // default to a single thread, because that is safe. + let fallback = NonZeroUsize::MIN; + // To limit startup overhead on massively parallel machines, don't use more + // than 64 threads. + let limit = NonZeroUsize::new(64).unwrap(); + + std::thread::available_parallelism() + .unwrap_or(fallback) + .min(limit) +} + +#[derive(Copy, Clone, PartialEq, Eq, ValueEnum)] +pub enum FileType { + #[value(alias = "f")] + File, + #[value(alias = "d", alias = "dir")] + Directory, + #[value(alias = "l")] + Symlink, + #[value(alias = "b")] + BlockDevice, + #[value(alias = "c")] + CharDevice, + /// A file which is executable by the current effective user + #[value(alias = "x")] + Executable, + #[value(alias = "e")] + Empty, + #[value(alias = "s")] + Socket, + #[value(alias = "p")] + Pipe, +} + +#[derive(Copy, Clone, PartialEq, Eq, Debug, ValueEnum)] +pub enum ColorWhen { + /// show colors if the output goes to an interactive console (default) + Auto, + /// always use colorized output + Always, + /// do not use colorized output + Never, +} + +#[derive(Copy, Clone, PartialEq, Eq, Debug, ValueEnum)] +pub enum StripCwdWhen { + /// Use the default behavior + Auto, + /// Always strip the ./ at the beginning of paths + Always, + /// Never strip the ./ + Never, +} + +#[derive(Copy, Clone, PartialEq, Eq, Debug, ValueEnum)] +pub enum HyperlinkWhen { + /// Use hyperlinks only if color is enabled + Auto, + /// Always use hyperlinks when printing file paths + Always, + /// Never use hyperlinks + Never, +} + +// there isn't a derive api for getting grouped values yet, +// so we have to use hand-rolled parsing for exec and exec-batch +pub struct Exec { + pub command: Option, +} + +impl clap::FromArgMatches for Exec { + fn from_arg_matches(matches: &ArgMatches) -> clap::error::Result { + let command = matches + .get_occurrences::("exec") + .map(CommandSet::new) + .or_else(|| { + matches + .get_occurrences::("exec_batch") + .map(CommandSet::new_batch) + }) + .transpose() + .map_err(|e| clap::Error::raw(ErrorKind::InvalidValue, e))?; + Ok(Self { command }) + } + + fn update_from_arg_matches(&mut self, matches: &ArgMatches) -> clap::error::Result<()> { + *self = Self::from_arg_matches(matches)?; + Ok(()) + } +} + +impl clap::Args for Exec { + fn augment_args(cmd: Command) -> Command { + cmd.arg(Arg::new("exec") + .action(ArgAction::Append) + .long("exec") + .short('x') + .num_args(1..) + .allow_hyphen_values(true) + .value_terminator(";") + .value_name("cmd") + .conflicts_with("list_details") + .help("Execute a command for each search result") + .long_help( + "Execute a command for each search result in parallel (use --threads=1 for sequential command execution). \ + The order in which different search results are processed and their output is printed is not guaranteed, even with --threads=1. \ + All positional arguments following --exec are considered to be arguments to the command - not to fd. \ + It is therefore recommended to place the '-x'/'--exec' option last. \ + Use '\\;' to terminate the command template if you need to continue passing fd arguments afterwards.\n\n\ + Specify this option multiple times to run several commands for each file or directory found. \ + For each file or directory, fd runs these commands one after another in the order they appear on the command line. \ + Terminate each command except the last with '\\;'. \ + When running in parallel, fd buffers command output and prints it together for each search result, \ + without interleaving it with command output for other results.\n\n\ + The following placeholders are substituted before the command is executed:\n \ + '{}': path (of the current search result)\n \ + '{/}': basename\n \ + '{//}': parent directory\n \ + '{.}': path without file extension\n \ + '{/.}': basename without file extension\n \ + '{{': literal '{' (for escaping)\n \ + '}}': literal '}' (for escaping)\n\n\ + If no placeholder is present, an implicit \"{}\" at the end is assumed.\n\n\ + Examples:\n\n \ + - find all *.zip files and unzip them:\n\n \ + fd -e zip -x unzip\n\n \ + - find *.h and *.cpp files and run \"clang-format -i ..\" for each of them:\n\n \ + fd -e h -e cpp -x clang-format -i\n\n \ + - search within `src/` and echo each match (place `-x` last):\n\n \ + fd . src -x echo\n\n \ + - Convert all *.jpg files to *.png files:\n\n \ + fd -e jpg -x convert {} {.}.png\ + ", + ), + ) + .arg( + Arg::new("exec_batch") + .action(ArgAction::Append) + .long("exec-batch") + .short('X') + .num_args(1..) + .allow_hyphen_values(true) + .value_terminator(";") + .value_name("cmd") + .conflicts_with_all(["exec", "list_details"]) + .help("Execute a command with all search results at once") + .long_help( + "Execute the given command once, with all search results as arguments.\n\ + The order of the arguments is non-deterministic, and should not be relied upon.\n\ + One of the following placeholders is substituted before the command is executed:\n \ + '{}': path (of all search results)\n \ + '{/}': basename\n \ + '{//}': parent directory\n \ + '{.}': path without file extension\n \ + '{/.}': basename without file extension\n \ + '{{': literal '{' (for escaping)\n \ + '}}': literal '}' (for escaping)\n\n\ + If no placeholder is present, an implicit \"{}\" at the end is assumed.\n\n\ + Examples:\n\n \ + - Find all test_*.py files and open them in your favorite editor:\n\n \ + fd -g 'test_*.py' -X vim\n\n \ + - Find all *.rs files and count the lines with \"wc -l ...\":\n\n \ + fd -e rs -X wc -l\ + " + ), + ) + } + + fn augment_args_for_update(cmd: Command) -> Command { + Self::augment_args(cmd) + } +} + +fn parse_millis(arg: &str) -> Result { + Ok(Duration::from_millis(arg.parse()?)) +} + +fn ensure_current_directory_exists(current_directory: &Path) -> anyhow::Result<()> { + if filesystem::is_existing_directory(current_directory) { + Ok(()) + } else { + Err(anyhow!( + "Could not retrieve current directory (has it been deleted?)." + )) + } +} diff --git a/userland/fd/src/config.rs b/userland/fd/src/config.rs new file mode 100644 index 0000000..58a1169 --- /dev/null +++ b/userland/fd/src/config.rs @@ -0,0 +1,140 @@ +use std::{path::PathBuf, sync::Arc, time::Duration}; + +use lscolors::LsColors; +use regex::bytes::RegexSet; + +use crate::exec::CommandSet; +use crate::filetypes::FileTypes; +#[cfg(unix)] +use crate::filter::OwnerFilter; +use crate::filter::{SizeFilter, TimeFilter}; +use crate::fmt::FormatTemplate; + +/// Configuration options for *fd*. +pub struct Config { + /// Cached current working directory for absolute path construction. + /// Populated when `--full-path` is set; `None` means search by filename only. + pub full_path_base: Option, + + /// Whether to ignore hidden files and directories (or not). + pub ignore_hidden: bool, + + /// Whether to respect `.fdignore` files or not. + pub read_fdignore: bool, + + /// Whether to respect ignore files in parent directories or not. + pub read_parent_ignore: bool, + + /// Whether to respect VCS ignore files (`.gitignore`, ..) or not. + pub read_vcsignore: bool, + + /// Whether to require a `.git` directory to respect gitignore files. + pub require_git_to_read_vcsignore: bool, + + /// Whether to respect the global ignore file or not. + pub read_global_ignore: bool, + + /// Whether to follow symlinks or not. + pub follow_links: bool, + + /// Whether to limit the search to starting file system or not. + pub one_file_system: bool, + + /// Whether elements of output should be separated by a null character + pub null_separator: bool, + + /// The maximum search depth, or `None` if no maximum search depth should be set. + /// + /// A depth of `1` includes all files under the current directory, a depth of `2` also includes + /// all files under subdirectories of the current directory, etc. + pub max_depth: Option, + + /// The minimum depth for reported entries, or `None`. + pub min_depth: Option, + + /// Whether to stop traversing into matching directories. + pub prune: bool, + + /// The number of threads to use. + pub threads: usize, + + /// If true, the program doesn't print anything and will instead return an exit code of 0 + /// if there's at least one match. Otherwise, the exit code will be 1. + pub quiet: bool, + + /// Time to buffer results internally before streaming to the console. This is useful to + /// provide a sorted output, in case the total execution time is shorter than + /// `max_buffer_time`. + pub max_buffer_time: Option, + + /// `None` if the output should not be colorized. Otherwise, a `LsColors` instance that defines + /// how to style different filetypes. + pub ls_colors: Option, + + /// Whether or not we are writing to an interactive terminal + #[cfg_attr(not(unix), allow(unused))] + pub interactive_terminal: bool, + + /// The type of file to search for. If set to `None`, all file types are displayed. If + /// set to `Some(..)`, only the types that are specified are shown. + pub file_types: Option, + + /// The extension to search for. Only entries matching the extension will be included. + /// + /// The value (if present) will be a lowercase string without leading dots. + pub extensions: Option, + + /// A format string to use to format results, similarly to exec + pub format: Option, + + /// If a value is supplied, each item found will be used to generate and execute commands. + pub command: Option>, + + /// Maximum number of search results to pass to each `command`. If zero, the number is + /// unlimited. + pub batch_size: usize, + + /// A list of glob patterns that should be excluded from the search. + pub exclude_patterns: Vec, + + /// A list of custom ignore files. + pub ignore_files: Vec, + + /// The given constraints on the size of returned files + pub size_constraints: Vec, + + /// Constraints on last modification time of files + pub time_constraints: Vec, + + #[cfg(unix)] + /// User/group ownership constraint + pub owner_constraint: Option, + + /// Whether or not to display filesystem errors + pub show_filesystem_errors: bool, + + /// The separator used to print file paths. + pub path_separator: Option, + + /// The actual separator, either the system default separator or `path_separator` + pub actual_path_separator: String, + + /// The maximum number of search results + pub max_results: Option, + + /// Whether or not to strip the './' prefix for search results + pub strip_cwd_prefix: bool, + + /// Whether or not to use hyperlinks on paths + pub hyperlink: bool, + + /// Names that should stop traversal down their parent. (e.g. https://bford.info/cachedir/). + pub ignore_contain: Vec, +} + +impl Config { + /// Check whether results are being printed. + pub fn is_printing(&self) -> bool { + self.command.is_none() + } +} diff --git a/userland/fd/src/dir_entry.rs b/userland/fd/src/dir_entry.rs new file mode 100644 index 0000000..93aa514 --- /dev/null +++ b/userland/fd/src/dir_entry.rs @@ -0,0 +1,191 @@ +use std::cell::OnceCell; +use std::ffi::OsString; +use std::fs::{FileType, Metadata}; +use std::path::{Path, PathBuf}; + +use lscolors::{Colorable, LsColors, Style}; + +use crate::config::Config; +use crate::filesystem::strip_current_dir; + +#[derive(Debug)] +enum DirEntryInner { + Normal(ignore::DirEntry), + // Broken symlinks reach us as walk errors rather than entries, so we carry + // over the depth the walker recorded on the error. + BrokenSymlink { path: PathBuf, depth: Option }, +} + +#[derive(Debug)] +pub struct DirEntry { + inner: DirEntryInner, + metadata: OnceCell>, + style: OnceCell>, +} + +impl DirEntry { + #[inline] + pub fn normal(e: ignore::DirEntry) -> Self { + Self { + inner: DirEntryInner::Normal(e), + metadata: OnceCell::new(), + style: OnceCell::new(), + } + } + + pub fn broken_symlink(path: PathBuf, depth: Option) -> Self { + Self { + inner: DirEntryInner::BrokenSymlink { path, depth }, + metadata: OnceCell::new(), + style: OnceCell::new(), + } + } + + pub fn path(&self) -> &Path { + match &self.inner { + DirEntryInner::Normal(e) => e.path(), + DirEntryInner::BrokenSymlink { path, .. } => path.as_path(), + } + } + + pub fn into_path(self) -> PathBuf { + match self.inner { + DirEntryInner::Normal(e) => e.into_path(), + DirEntryInner::BrokenSymlink { path, .. } => path, + } + } + + /// Returns the path as it should be presented to the user. + /// When stripping `./` would leave the path starting with `-`, keep the original + /// (with `./`) so downstream tools don't interpret it as an option. + pub fn stripped_path(&self, config: &Config) -> &Path { + let path = self.path(); + if config.strip_cwd_prefix { + let stripped = strip_current_dir(path); + if starts_with_dash(stripped) { + path + } else { + stripped + } + } else { + path + } + } + + /// Returns the path as it should be presented to the user. + pub fn into_stripped_path(self, config: &Config) -> PathBuf { + if config.strip_cwd_prefix { + self.stripped_path(config).to_path_buf() + } else { + self.into_path() + } + } + + pub fn file_type(&self) -> Option { + match &self.inner { + DirEntryInner::Normal(e) => e.file_type(), + DirEntryInner::BrokenSymlink { .. } => self.metadata().map(|m| m.file_type()), + } + } + + pub fn metadata(&self) -> Option<&Metadata> { + self.metadata + .get_or_init(|| match &self.inner { + DirEntryInner::Normal(e) => e.metadata().ok(), + DirEntryInner::BrokenSymlink { path, .. } => path.symlink_metadata().ok(), + }) + .as_ref() + } + + pub fn depth(&self) -> Option { + match &self.inner { + DirEntryInner::Normal(e) => Some(e.depth()), + DirEntryInner::BrokenSymlink { depth, .. } => *depth, + } + } + + pub fn style(&self, ls_colors: &LsColors) -> Option<&Style> { + self.style + .get_or_init(|| ls_colors.style_for(self).cloned()) + .as_ref() + } +} + +fn starts_with_dash(path: &Path) -> bool { + path.as_os_str().as_encoded_bytes().first() == Some(&b'-') +} + +impl PartialEq for DirEntry { + #[inline] + fn eq(&self, other: &Self) -> bool { + self.path() == other.path() + } +} + +impl Eq for DirEntry {} + +impl PartialOrd for DirEntry { + #[inline] + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl Ord for DirEntry { + #[inline] + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + self.path().cmp(other.path()) + } +} + +impl Colorable for DirEntry { + fn path(&self) -> PathBuf { + self.path().to_owned() + } + + fn file_name(&self) -> OsString { + let name = match &self.inner { + DirEntryInner::Normal(e) => e.file_name(), + DirEntryInner::BrokenSymlink { path, .. } => { + // Path::file_name() only works if the last component is Normal, + // but we want it for all component types, so we open code it. + // Copied from LsColors::style_for_path_with_metadata(). + path.components() + .next_back() + .map(|c| c.as_os_str()) + .unwrap_or_else(|| path.as_os_str()) + } + }; + name.to_owned() + } + + fn file_type(&self) -> Option { + self.file_type() + } + + fn metadata(&self) -> Option { + self.metadata().cloned() + } +} + +#[cfg(test)] +mod tests { + use super::starts_with_dash; + use std::path::Path; + + #[test] + fn dash_prefixed_paths_detected() { + assert!(starts_with_dash(Path::new("-rf"))); + assert!(starts_with_dash(Path::new("--delete"))); + assert!(starts_with_dash(Path::new("-"))); + } + + #[test] + fn safe_paths_not_flagged() { + assert!(!starts_with_dash(Path::new("foo"))); + assert!(!starts_with_dash(Path::new("./foo"))); + assert!(!starts_with_dash(Path::new("sub/-rf"))); + assert!(!starts_with_dash(Path::new(""))); + assert!(!starts_with_dash(Path::new(" -rf"))); + } +} diff --git a/userland/fd/src/error.rs b/userland/fd/src/error.rs new file mode 100644 index 0000000..bec1b44 --- /dev/null +++ b/userland/fd/src/error.rs @@ -0,0 +1,51 @@ +use std::fmt::Display; +use std::io; + +use crate::sanitize::write_sanitized; + +/// Print an error message using a format string, sanitizing any arguments if necessary +macro_rules! print_error { + ($fmt:literal, $($arg:expr),*) => { + eprintln!(concat!("[fd error]: ", $fmt), $($crate::error::SanitizeErr::sanitize($arg)),*) + } +} + +/// Trait for specifying how to sanitize a type for error display, if needed. +pub(crate) trait SanitizeErr { + type Sanitized: Display; + + fn sanitize(self) -> Self::Sanitized; +} + +impl SanitizeErr for &str { + type Sanitized = Self; + + fn sanitize(self) -> Self { + self + } +} + +impl SanitizeErr for io::Error { + type Sanitized = SanitizedError; + + fn sanitize(self) -> Self::Sanitized { + SanitizedError(self) + } +} + +impl SanitizeErr for ignore::Error { + type Sanitized = SanitizedError; + + fn sanitize(self) -> Self::Sanitized { + SanitizedError(self) + } +} + +pub struct SanitizedError(E); + +impl Display for SanitizedError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let raw = self.0.to_string(); + write_sanitized(f, &raw) + } +} diff --git a/userland/fd/src/exec/command.rs b/userland/fd/src/exec/command.rs new file mode 100644 index 0000000..0e08133 --- /dev/null +++ b/userland/fd/src/exec/command.rs @@ -0,0 +1,114 @@ +use std::io; +use std::io::Write; + +use argmax::Command; + +use crate::exit_codes::ExitCode; + +struct Outputs { + stdout: Vec, + stderr: Vec, +} +pub struct OutputBuffer { + null_separator: bool, + outputs: Vec, +} + +impl OutputBuffer { + pub fn new(null_separator: bool) -> Self { + Self { + null_separator, + outputs: Vec::new(), + } + } + + fn push(&mut self, stdout: Vec, stderr: Vec) { + self.outputs.push(Outputs { stdout, stderr }); + } + + fn write(self) { + // Avoid taking the lock if there is nothing to do. + // If null_separator is true, then we still need to write the + // null separator, because the output may have been written directly + // to stdout + if self.outputs.is_empty() && !self.null_separator { + return; + } + + let stdout = io::stdout(); + let stderr = io::stderr(); + + // While we hold these locks, only this thread will be able + // to write its outputs. + let mut stdout = stdout.lock(); + let mut stderr = stderr.lock(); + + for output in self.outputs.iter() { + let _ = stdout.write_all(&output.stdout); + let _ = stderr.write_all(&output.stderr); + } + if self.null_separator { + // If null_separator is enabled, then we should write a \0 at the end + // of the output for this entry + let _ = stdout.write_all(b"\0"); + } + } +} + +/// Executes a command. +pub fn execute_commands>>( + cmds: I, + mut output_buffer: OutputBuffer, + enable_output_buffering: bool, +) -> ExitCode { + for result in cmds { + let mut cmd = match result { + Ok(cmd) => cmd, + Err(e) => return handle_cmd_error(None, e), + }; + + // Spawn the supplied command. + let output = if enable_output_buffering { + cmd.output() + } else { + // If running on only one thread, don't buffer output + // Allows for viewing and interacting with intermediate command output + cmd.spawn().and_then(|c| c.wait_with_output()) + }; + + // Then wait for the command to exit, if it was spawned. + match output { + Ok(output) => { + if enable_output_buffering { + output_buffer.push(output.stdout, output.stderr); + } + if output.status.code() != Some(0) { + output_buffer.write(); + return ExitCode::GeneralError; + } + } + Err(why) => { + output_buffer.write(); + return handle_cmd_error(Some(&cmd), why); + } + } + } + output_buffer.write(); + ExitCode::Success +} + +pub fn handle_cmd_error(cmd: Option<&Command>, err: io::Error) -> ExitCode { + match (cmd, err) { + (Some(cmd), err) if err.kind() == io::ErrorKind::NotFound => { + print_error!( + "Command not found: {:?}", + cmd.get_program().to_string_lossy().as_ref() + ); + ExitCode::GeneralError + } + (_, err) => { + print_error!("Problem while executing command: {}", err); + ExitCode::GeneralError + } + } +} diff --git a/userland/fd/src/exec/job.rs b/userland/fd/src/exec/job.rs new file mode 100644 index 0000000..41baa43 --- /dev/null +++ b/userland/fd/src/exec/job.rs @@ -0,0 +1,63 @@ +use crate::config::Config; +use crate::exit_codes::{ExitCode, merge_exitcodes}; +use crate::walk::WorkerResult; + +use super::CommandSet; + +/// An event loop that listens for inputs from the `rx` receiver. Each received input will +/// generate a command with the supplied command template. The generated command will then +/// be executed, and this process will continue until the receiver's sender has closed. +pub fn job( + results: impl IntoIterator, + cmd: &CommandSet, + config: &Config, +) -> ExitCode { + // Output should be buffered when only running a single thread + let buffer_output: bool = config.threads > 1; + + let mut ret = ExitCode::Success; + for result in results { + // Obtain the next result from the receiver, else if the channel + // has closed, exit from the loop + let dir_entry = match result { + WorkerResult::Entry(dir_entry) => dir_entry, + WorkerResult::Error(err) => { + if config.show_filesystem_errors { + print_error!("{}", err); + } + continue; + } + }; + + // Generate a command, execute it and store its exit code. + let code = cmd.execute( + dir_entry.stripped_path(config), + config.path_separator.as_deref(), + config.null_separator, + buffer_output, + ); + ret = merge_exitcodes([ret, code]); + } + // Returns error in case of any error. + ret +} + +pub fn batch( + results: impl IntoIterator, + cmd: &CommandSet, + config: &Config, +) -> ExitCode { + let paths = results + .into_iter() + .filter_map(|worker_result| match worker_result { + WorkerResult::Entry(dir_entry) => Some(dir_entry.into_stripped_path(config)), + WorkerResult::Error(err) => { + if config.show_filesystem_errors { + print_error!("{}", err); + } + None + } + }); + + cmd.execute_batch(paths, config.batch_size, config.path_separator.as_deref()) +} diff --git a/userland/fd/src/exec/mod.rs b/userland/fd/src/exec/mod.rs new file mode 100644 index 0000000..134ecf2 --- /dev/null +++ b/userland/fd/src/exec/mod.rs @@ -0,0 +1,486 @@ +mod command; +mod job; + +use std::ffi::OsString; +use std::io; +use std::iter; +use std::path::{Path, PathBuf}; +use std::process::Stdio; + +use anyhow::{Result, bail}; +use argmax::Command; + +use crate::exec::command::OutputBuffer; +use crate::exit_codes::{ExitCode, merge_exitcodes}; +use crate::fmt::{FormatTemplate, Token}; + +use self::command::{execute_commands, handle_cmd_error}; +pub use self::job::{batch, job}; + +/// Execution mode of the command +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ExecutionMode { + /// Command is executed for each search result + OneByOne, + /// Command is run for a batch of results at once + Batch, +} + +#[derive(Debug, Clone, PartialEq)] +pub struct CommandSet { + mode: ExecutionMode, + commands: Vec, +} + +impl CommandSet { + pub fn new(input: I) -> Result + where + I: IntoIterator, + T: IntoIterator, + S: AsRef, + { + Ok(Self { + mode: ExecutionMode::OneByOne, + commands: input + .into_iter() + .map(|args| CommandTemplate::new(args, ExecutionMode::OneByOne)) + .collect::>()?, + }) + } + + pub fn new_batch(input: I) -> Result + where + I: IntoIterator, + T: IntoIterator, + S: AsRef, + { + Ok(Self { + mode: ExecutionMode::Batch, + commands: input + .into_iter() + .map(|args| { + let cmd = CommandTemplate::new(args, ExecutionMode::Batch)?; + if cmd.number_of_tokens() > 1 { + bail!("Only one placeholder allowed for batch commands"); + } + Ok(cmd) + }) + .collect::>>()?, + }) + } + + pub fn in_batch_mode(&self) -> bool { + self.mode == ExecutionMode::Batch + } + + pub fn execute( + &self, + input: &Path, + path_separator: Option<&str>, + null_separator: bool, + buffer_output: bool, + ) -> ExitCode { + let commands = self + .commands + .iter() + .map(|c| c.generate(input, path_separator)); + execute_commands(commands, OutputBuffer::new(null_separator), buffer_output) + } + + pub fn execute_batch(&self, paths: I, limit: usize, path_separator: Option<&str>) -> ExitCode + where + I: Iterator, + { + let builders: io::Result> = self + .commands + .iter() + .map(|c| CommandBuilder::new(c, limit)) + .collect(); + + match builders { + Ok(mut builders) => { + for path in paths { + for builder in &mut builders { + if let Err(e) = builder.push(&path, path_separator) { + return handle_cmd_error(Some(&builder.cmd), e); + } + } + } + + for builder in &mut builders { + if let Err(e) = builder.finish() { + return handle_cmd_error(Some(&builder.cmd), e); + } + } + + merge_exitcodes(builders.iter().map(|b| b.exit_code())) + } + Err(e) => handle_cmd_error(None, e), + } + } +} + +/// Represents a multi-exec command as it is built. +#[derive(Debug)] +struct CommandBuilder { + pre_args: Vec, + path_arg: FormatTemplate, + post_args: Vec, + cmd: Command, + count: usize, + limit: usize, + exit_code: ExitCode, +} + +impl CommandBuilder { + fn new(template: &CommandTemplate, limit: usize) -> io::Result { + let mut pre_args = vec![]; + let mut path_arg = None; + let mut post_args = vec![]; + + for arg in &template.args { + if arg.has_tokens() { + path_arg = Some(arg.clone()); + } else if path_arg.is_none() { + pre_args.push(arg.generate("", None)); + } else { + post_args.push(arg.generate("", None)); + } + } + + let cmd = Self::new_command(&pre_args)?; + + Ok(Self { + pre_args, + path_arg: path_arg.unwrap(), + post_args, + cmd, + count: 0, + limit, + exit_code: ExitCode::Success, + }) + } + + fn new_command(pre_args: &[OsString]) -> io::Result { + let mut cmd = Command::new(&pre_args[0]); + cmd.stdin(Stdio::inherit()); + cmd.stdout(Stdio::inherit()); + cmd.stderr(Stdio::inherit()); + cmd.try_args(&pre_args[1..])?; + Ok(cmd) + } + + fn push(&mut self, path: &Path, separator: Option<&str>) -> io::Result<()> { + if self.limit > 0 && self.count >= self.limit { + self.finish()?; + } + + let arg = self.path_arg.generate(path, separator); + if !self + .cmd + .args_would_fit(iter::once(&arg).chain(&self.post_args)) + { + self.finish()?; + } + + self.cmd.try_arg(arg)?; + self.count += 1; + Ok(()) + } + + fn finish(&mut self) -> io::Result<()> { + if self.count > 0 { + self.cmd.try_args(&self.post_args)?; + if !self.cmd.status()?.success() { + self.exit_code = ExitCode::GeneralError; + } + + self.cmd = Self::new_command(&self.pre_args)?; + self.count = 0; + } + + Ok(()) + } + + fn exit_code(&self) -> ExitCode { + self.exit_code + } +} + +/// Represents a template that is utilized to generate command strings. +/// +/// The template is meant to be coupled with an input in order to generate a command. The +/// `generate_and_execute()` method will be used to generate a command and execute it. +#[derive(Debug, Clone, PartialEq)] +struct CommandTemplate { + args: Vec, +} + +impl CommandTemplate { + fn new(input: I, mode: ExecutionMode) -> Result + where + I: IntoIterator, + S: AsRef, + { + let mut args = Vec::new(); + let mut has_placeholder = false; + + for arg in input { + let arg = arg.as_ref(); + + let tmpl = FormatTemplate::parse(arg); + has_placeholder |= tmpl.has_tokens(); + args.push(tmpl); + } + + // We need to check that we have at least one argument, because if not + // it will try to execute each file and directory it finds. + // + // Sadly, clap can't currently handle this for us, see + // https://github.com/clap-rs/clap/issues/3542 + if args.is_empty() { + bail!("No executable provided for --exec or --exec-batch"); + } + + // A placeholder as the executable is meaningful for `--exec` but never for `--exec-batch`. + if mode == ExecutionMode::Batch && args[0].has_tokens() { + bail!("First argument of --exec-batch must be a fixed executable, not a placeholder"); + } + + // If a placeholder token was not supplied, append one at the end of the command. + if !has_placeholder { + args.push(FormatTemplate::Tokens(vec![Token::Placeholder])); + } + + Ok(Self { args }) + } + + fn number_of_tokens(&self) -> usize { + self.args.iter().filter(|arg| arg.has_tokens()).count() + } + + /// Generates and executes a command. + /// + /// Using the internal `args` field, and a supplied `input` variable, a `Command` will be + /// build. + fn generate(&self, input: &Path, path_separator: Option<&str>) -> io::Result { + let mut cmd = Command::new(self.args[0].generate(input, path_separator)); + for arg in &self.args[1..] { + cmd.try_arg(arg.generate(input, path_separator))?; + } + Ok(cmd) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn generate_str(template: &CommandTemplate, input: &str) -> Vec { + template + .args + .iter() + .map(|arg| arg.generate(input, None).into_string().unwrap()) + .collect() + } + + #[test] + fn tokens_with_placeholder() { + assert_eq!( + CommandSet::new(vec![vec![&"echo", &"${SHELL}:"]]).unwrap(), + CommandSet { + commands: vec![CommandTemplate { + args: vec![ + FormatTemplate::Text("echo".into()), + FormatTemplate::Text("${SHELL}:".into()), + FormatTemplate::Tokens(vec![Token::Placeholder]), + ] + }], + mode: ExecutionMode::OneByOne, + } + ); + } + + #[test] + fn tokens_with_no_extension() { + assert_eq!( + CommandSet::new(vec![vec!["echo", "{.}"]]).unwrap(), + CommandSet { + commands: vec![CommandTemplate { + args: vec![ + FormatTemplate::Text("echo".into()), + FormatTemplate::Tokens(vec![Token::NoExt]), + ], + }], + mode: ExecutionMode::OneByOne, + } + ); + } + + #[test] + fn tokens_with_basename() { + assert_eq!( + CommandSet::new(vec![vec!["echo", "{/}"]]).unwrap(), + CommandSet { + commands: vec![CommandTemplate { + args: vec![ + FormatTemplate::Text("echo".into()), + FormatTemplate::Tokens(vec![Token::Basename]), + ], + }], + mode: ExecutionMode::OneByOne, + } + ); + } + + #[test] + fn tokens_with_parent() { + assert_eq!( + CommandSet::new(vec![vec!["echo", "{//}"]]).unwrap(), + CommandSet { + commands: vec![CommandTemplate { + args: vec![ + FormatTemplate::Text("echo".into()), + FormatTemplate::Tokens(vec![Token::Parent]), + ], + }], + mode: ExecutionMode::OneByOne, + } + ); + } + + #[test] + fn tokens_with_basename_no_extension() { + assert_eq!( + CommandSet::new(vec![vec!["echo", "{/.}"]]).unwrap(), + CommandSet { + commands: vec![CommandTemplate { + args: vec![ + FormatTemplate::Text("echo".into()), + FormatTemplate::Tokens(vec![Token::BasenameNoExt]), + ], + }], + mode: ExecutionMode::OneByOne, + } + ); + } + + #[test] + fn tokens_with_literal_braces() { + let template = + CommandTemplate::new(vec!["{{}}", "{{", "{.}}"], ExecutionMode::OneByOne).unwrap(); + assert_eq!( + generate_str(&template, "foo"), + vec!["{}", "{", "{.}", "foo"] + ); + } + + #[test] + fn tokens_with_literal_braces_and_placeholder() { + let template = + CommandTemplate::new(vec!["echo", "{{{},end}"], ExecutionMode::OneByOne).unwrap(); + assert_eq!(generate_str(&template, "foo"), vec!["echo", "{foo,end}"]); + } + + #[test] + fn tokens_multiple() { + assert_eq!( + CommandSet::new(vec![vec!["cp", "{}", "{/.}.ext"]]).unwrap(), + CommandSet { + commands: vec![CommandTemplate { + args: vec![ + FormatTemplate::Text("cp".into()), + FormatTemplate::Tokens(vec![Token::Placeholder]), + FormatTemplate::Tokens(vec![ + Token::BasenameNoExt, + Token::Text(".ext".into()) + ]), + ], + }], + mode: ExecutionMode::OneByOne, + } + ); + } + + #[test] + fn tokens_single_batch() { + assert_eq!( + CommandSet::new_batch(vec![vec!["echo", "{.}"]]).unwrap(), + CommandSet { + commands: vec![CommandTemplate { + args: vec![ + FormatTemplate::Text("echo".into()), + FormatTemplate::Tokens(vec![Token::NoExt]), + ], + }], + mode: ExecutionMode::Batch, + } + ); + } + + #[test] + fn tokens_multiple_batch() { + assert!(CommandSet::new_batch(vec![vec!["echo", "{.}", "{}"]]).is_err()); + } + + #[test] + fn template_no_args() { + assert!( + CommandTemplate::new::, &'static str>(vec![], ExecutionMode::OneByOne).is_err() + ); + } + + #[test] + fn command_set_no_args() { + assert!(CommandSet::new(vec![vec!["echo"], vec![]]).is_err()); + } + + #[test] + fn placeholder_as_executable_rejected() { + assert!(CommandSet::new(vec![vec!["{}"]]).is_ok()); + assert!(CommandSet::new(vec![vec!["{/}", "arg"]]).is_ok()); + assert!(CommandSet::new_batch(vec![vec!["{}"]]).is_err()); + } + + #[test] + fn generate_custom_path_separator() { + let arg = FormatTemplate::Tokens(vec![Token::Placeholder]); + macro_rules! check { + ($input:expr, $expected:expr) => { + assert_eq!(arg.generate($input, Some("#")), OsString::from($expected)); + }; + } + + check!("foo", "foo"); + check!("foo/bar", "foo#bar"); + check!("/foo/bar/baz", "#foo#bar#baz"); + } + + #[cfg(windows)] + #[test] + fn generate_custom_path_separator_windows() { + let arg = FormatTemplate::Tokens(vec![Token::Placeholder]); + macro_rules! check { + ($input:expr, $expected:expr) => { + assert_eq!(arg.generate($input, Some("#")), OsString::from($expected)); + }; + } + + // path starting with a drive letter + check!(r"C:\foo\bar", "C:#foo#bar"); + // UNC path + check!(r"\\server\share\path", "##server#share#path"); + // Drive Relative path - no separator after the colon omits the RootDir path component. + // This is uncommon, but valid + check!(r"C:foo\bar", "C:foo#bar"); + + // forward slashes should get normalized and interpreted as separators + check!("C:/foo/bar", "C:#foo#bar"); + check!("C:foo/bar", "C:foo#bar"); + + // Rust does not interpret "//server/share" as a UNC path, but rather as a normal + // absolute path that begins with RootDir, and the two slashes get combined together as + // a single path separator during normalization. + //check!("//server/share/path", "##server#share#path"); + } +} diff --git a/userland/fd/src/exit_codes.rs b/userland/fd/src/exit_codes.rs new file mode 100644 index 0000000..9ab4b84 --- /dev/null +++ b/userland/fd/src/exit_codes.rs @@ -0,0 +1,94 @@ +use std::process; + +#[cfg(unix)] +use nix::sys::signal::{SigHandler, Signal, raise, signal}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ExitCode { + Success, + HasResults(bool), + GeneralError, + KilledBySigint, +} + +impl From for i32 { + fn from(code: ExitCode) -> Self { + match code { + ExitCode::Success => 0, + ExitCode::HasResults(has_results) => !has_results as Self, + ExitCode::GeneralError => 1, + ExitCode::KilledBySigint => 130, + } + } +} + +impl ExitCode { + fn is_error(self) -> bool { + i32::from(self) != 0 + } + + /// Exit the process with the appropriate code. + pub fn exit(self) -> ! { + #[cfg(unix)] + if self == Self::KilledBySigint { + // Get rid of the SIGINT handler, if present, and raise SIGINT + unsafe { + if signal(Signal::SIGINT, SigHandler::SigDfl).is_ok() { + let _ = raise(Signal::SIGINT); + } + } + } + + process::exit(self.into()) + } +} + +pub fn merge_exitcodes(results: impl IntoIterator) -> ExitCode { + if results.into_iter().any(ExitCode::is_error) { + return ExitCode::GeneralError; + } + ExitCode::Success +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn success_when_no_results() { + assert_eq!(merge_exitcodes([]), ExitCode::Success); + } + + #[test] + fn general_error_if_at_least_one_error() { + assert_eq!( + merge_exitcodes([ExitCode::GeneralError]), + ExitCode::GeneralError + ); + assert_eq!( + merge_exitcodes([ExitCode::KilledBySigint]), + ExitCode::GeneralError + ); + assert_eq!( + merge_exitcodes([ExitCode::KilledBySigint, ExitCode::Success]), + ExitCode::GeneralError + ); + assert_eq!( + merge_exitcodes([ExitCode::Success, ExitCode::GeneralError]), + ExitCode::GeneralError + ); + assert_eq!( + merge_exitcodes([ExitCode::GeneralError, ExitCode::KilledBySigint]), + ExitCode::GeneralError + ); + } + + #[test] + fn success_if_no_error() { + assert_eq!(merge_exitcodes([ExitCode::Success]), ExitCode::Success); + assert_eq!( + merge_exitcodes([ExitCode::Success, ExitCode::Success]), + ExitCode::Success + ); + } +} diff --git a/userland/fd/src/filesystem.rs b/userland/fd/src/filesystem.rs new file mode 100644 index 0000000..e2d6026 --- /dev/null +++ b/userland/fd/src/filesystem.rs @@ -0,0 +1,320 @@ +use std::env; +use std::fs; +use std::io; +#[cfg(any(unix, target_os = "redox"))] +use std::os::unix::fs::FileTypeExt; +use std::path::{Path, PathBuf}; + +use normpath::PathExt; + +use crate::dir_entry; + +pub fn path_absolute_form(path: &Path) -> io::Result { + if path.is_absolute() { + return Ok(path.to_path_buf()); + } + + let path = path.strip_prefix(".").unwrap_or(path); + env::current_dir().map(|path_buf| path_buf.join(path)) +} + +pub fn absolute_path(path: &Path) -> io::Result { + let path_buf = path_absolute_form(path)?; + + #[cfg(windows)] + let path_buf = strip_verbatim_prefix(&path_buf); + + Ok(path_buf) +} + +/// Remove the `\\?\` verbatim prefix from a path without turning it into a +/// relative path. +/// +/// The prefix is dropped by rewriting the path *component by component*, so +/// that the meaning of the remaining components is preserved: +/// +/// * `\\?\C:\foo` -> `C:\foo` (verbatim disk path) +/// * `\\?\UNC\server\share\foo` -> `\\server\share\foo` (verbatim UNC path) +/// +/// Verbatim paths that cannot be expressed without the prefix (for example +/// `\\?\Volume{GUID}\foo`, where `Volume{GUID}` is a device name and not a +/// relative file name) are returned unchanged, and so are all non-verbatim +/// paths such as `\\server\share\foo` or `\\.\COM1`. +/// +/// Unlike the previous string trimming, this never goes through +/// `to_string_lossy()`, so paths that are not valid UTF-8 survive unchanged. +#[cfg(windows)] +fn strip_verbatim_prefix(path: &Path) -> PathBuf { + use std::ffi::OsString; + use std::path::{Component, Prefix}; + + let mut components = path.components(); + let Some(Component::Prefix(prefix)) = components.next() else { + return path.to_path_buf(); + }; + + let has_root = matches!(components.clone().next(), Some(Component::RootDir)); + let mut base = OsString::new(); + // `Prefix::Disk` / `Prefix::UNC` cannot rebuild this prefix: a `Prefix` is a + // parse label, not the bytes of the path, and `PrefixComponent` has no stable + // constructor. Writing it out by hand is the only way to drop `\\?\`. + match prefix.kind() { + Prefix::VerbatimDisk(drive) => { + base.push(format!("{}:", drive as char)); + if has_root { + base.push(r"\"); + } + } + Prefix::VerbatimUNC(server, share) => { + base.push(r"\\"); + base.push(server); + base.push(r"\"); + base.push(share); + if has_root { + base.push(r"\"); + } + } + // `Prefix::Verbatim(_)` (device paths like `\\?\Volume{GUID}`) and + // `Prefix::DeviceNS(_)` cannot be written without their prefix. + _ => return path.to_path_buf(), + } + + if has_root { + // The root separator is already part of `base`, and `join` would append + // it again; skipping the root component keeps the prefix we just built + // instead of letting `join` replace it. + let _ = components.next(); + } + if components.as_path().is_empty() { + // `\\?\C:` and `\\?\UNC\server\share`: the verbatim form is the whole + // path, and joining an empty tail would append a separator the input + // did not have. + return PathBuf::from(base); + } + Path::new(&base).join(components) +} + +pub fn is_existing_directory(path: &Path) -> bool { + // Note: we do not use `.exists()` here, as `.` always exists, even if + // the CWD has been deleted. + path.is_dir() && (path.file_name().is_some() || path.normalize().is_ok()) +} + +pub fn is_empty(entry: &dir_entry::DirEntry) -> bool { + if let Some(file_type) = entry.file_type() { + if file_type.is_dir() { + if let Ok(mut entries) = fs::read_dir(entry.path()) { + entries.next().is_none() + } else { + false + } + } else if file_type.is_file() { + entry.metadata().map(|m| m.len() == 0).unwrap_or(false) + } else { + false + } + } else { + false + } +} + +#[cfg(any(unix, target_os = "redox"))] +pub fn is_block_device(ft: fs::FileType) -> bool { + ft.is_block_device() +} + +#[cfg(any(windows, target_os = "wasi"))] +pub fn is_block_device(_: fs::FileType) -> bool { + false +} + +#[cfg(any(unix, target_os = "redox"))] +pub fn is_char_device(ft: fs::FileType) -> bool { + ft.is_char_device() +} + +#[cfg(any(windows, target_os = "wasi"))] +pub fn is_char_device(_: fs::FileType) -> bool { + false +} + +#[cfg(any(unix, target_os = "redox"))] +pub fn is_socket(ft: fs::FileType) -> bool { + ft.is_socket() +} + +#[cfg(any(windows, target_os = "wasi"))] +pub fn is_socket(_: fs::FileType) -> bool { + false +} + +#[cfg(any(unix, target_os = "redox"))] +pub fn is_pipe(ft: fs::FileType) -> bool { + ft.is_fifo() +} + +#[cfg(any(windows, target_os = "wasi"))] +pub fn is_pipe(_: fs::FileType) -> bool { + false +} + +/// Remove the `./` prefix from a path. +pub fn strip_current_dir(path: &Path) -> &Path { + path.strip_prefix(".").unwrap_or(path) +} + +/// Default value for the path_separator, mainly for MSYS/MSYS2, which set the MSYSTEM +/// environment variable, and we set fd's path separator to '/' rather than Rust's default of '\'. +/// +/// Returns Some to use a nonstandard path separator, or None to use rust's default on the target +/// platform. +pub fn default_path_separator() -> Option { + if cfg!(windows) { + let msystem = env::var("MSYSTEM").ok()?; + if !msystem.is_empty() { + return Some("/".to_owned()); + } + } + None +} + +#[cfg(test)] +mod tests { + use super::strip_current_dir; + use std::path::Path; + + #[test] + fn strip_current_dir_basic() { + assert_eq!(strip_current_dir(Path::new("./foo")), Path::new("foo")); + assert_eq!(strip_current_dir(Path::new("foo")), Path::new("foo")); + assert_eq!( + strip_current_dir(Path::new("./foo/bar/baz")), + Path::new("foo/bar/baz") + ); + assert_eq!( + strip_current_dir(Path::new("foo/bar/baz")), + Path::new("foo/bar/baz") + ); + } +} + +#[cfg(all(test, windows))] +mod windows_tests { + use super::{absolute_path, strip_verbatim_prefix}; + use std::ffi::OsString; + use std::os::windows::ffi::OsStringExt; + use std::path::{Path, PathBuf}; + + #[test] + fn strip_verbatim_prefix_verbatim_disk() { + assert_eq!( + strip_verbatim_prefix(Path::new(r"\\?\C:\foo\bar")), + Path::new(r"C:\foo\bar") + ); + // A bare drive root keeps its root separator. + assert_eq!( + strip_verbatim_prefix(Path::new(r"\\?\C:\")), + Path::new(r"C:\") + ); + // Drive-relative verbatim path (`\\?\C:`) must not gain a root. + assert_eq!( + strip_verbatim_prefix(Path::new(r"\\?\C:")), + Path::new(r"C:") + ); + } + + #[test] + fn strip_verbatim_prefix_verbatim_unc() { + let stripped = strip_verbatim_prefix(Path::new(r"\\?\UNC\server\share\foo")); + assert_eq!(stripped, Path::new(r"\\server\share\foo")); + // The whole point of the bug: the result must stay absolute. + assert!(stripped.is_absolute()); + assert_eq!( + stripped.components().next(), + Path::new(r"\\server\share\foo").components().next() + ); + + // The shape produced by the old `trim_start_matches(r"\\?\")` code. + assert_ne!(stripped, Path::new(r"UNC\server\share\foo")); + assert_ne!(stripped, Path::new(r"\server\share\foo")); + } + + #[test] + fn strip_verbatim_prefix_verbatim_unc_without_trailing_path() { + assert_eq!( + strip_verbatim_prefix(Path::new(r"\\?\UNC\server\share")), + Path::new(r"\\server\share") + ); + // ... and keeps the exact bytes: joining an empty tail would have + // appended a separator the input did not have. + assert_eq!( + strip_verbatim_prefix(Path::new(r"\\?\UNC\server\share")) + .as_os_str() + .as_encoded_bytes(), + Path::new(r"\\server\share").as_os_str().as_encoded_bytes() + ); + assert_eq!( + strip_verbatim_prefix(Path::new(r"\\?\UNC\server\share\")), + Path::new(r"\\server\share\") + ); + } + + #[test] + fn strip_verbatim_prefix_leaves_other_paths_alone() { + // Verbatim device paths cannot be expressed without the prefix. + let volume = Path::new(r"\\?\Volume{2b1e7a6b-1f2c-4c66-8c47-9b0e0e2f3a11}\foo"); + assert_eq!(strip_verbatim_prefix(volume), volume); + let pipe = Path::new(r"\\?\pipe\fd"); + assert_eq!(strip_verbatim_prefix(pipe), pipe); + + // Non-verbatim paths are untouched. + for path in [ + r"C:\foo\bar", + r"\\server\share\foo", + r"\\.\COM1", + r"C:", + r"foo\bar", + ] { + assert_eq!(strip_verbatim_prefix(Path::new(path)), Path::new(path)); + } + } + + #[test] + fn absolute_path_keeps_unc_absolute() { + assert_eq!( + absolute_path(Path::new(r"\\?\UNC\server\share\foo")).unwrap(), + Path::new(r"\\server\share\foo") + ); + assert_eq!( + absolute_path(Path::new(r"\\?\C:\foo")).unwrap(), + Path::new(r"C:\foo") + ); + assert_eq!( + absolute_path(Path::new(r"\\server\share\foo")).unwrap(), + Path::new(r"\\server\share\foo") + ); + } + + #[test] + fn absolute_path_preserves_non_utf8_paths() { + // A lone surrogate is not valid UTF-8 but is a valid UTF-16 code unit, + // and therefore a valid Windows path component. `to_string_lossy()` + // used to replace it with U+FFFD. + let mut wide: Vec = r"\\?\D:\".encode_utf16().collect(); + wide.push(0xD800); + wide.extend("x".encode_utf16()); + let path = PathBuf::from(OsString::from_wide(&wide)); + + let expected = { + let mut wide: Vec = r"D:\".encode_utf16().collect(); + wide.push(0xD800); + wide.extend("x".encode_utf16()); + PathBuf::from(OsString::from_wide(&wide)) + }; + assert_eq!(absolute_path(&path).unwrap(), expected); + assert_eq!( + absolute_path(&path).unwrap().as_os_str().as_encoded_bytes(), + expected.as_os_str().as_encoded_bytes() + ); + } +} diff --git a/userland/fd/src/filetypes.rs b/userland/fd/src/filetypes.rs new file mode 100644 index 0000000..669668c --- /dev/null +++ b/userland/fd/src/filetypes.rs @@ -0,0 +1,111 @@ +use crate::cli::FileType; +use crate::dir_entry; +use crate::filesystem; + +use faccess::PathExt; + +/// Whether or not to show +#[derive(Default, Debug, PartialEq, Eq)] +pub struct FileTypes { + pub files: bool, + pub directories: bool, + pub symlinks: bool, + pub block_devices: bool, + pub char_devices: bool, + pub sockets: bool, + pub pipes: bool, + pub executables_only: bool, + pub empty_only: bool, +} + +impl FileTypes { + pub fn from_types(types: &[FileType]) -> Self { + let mut file_types = FileTypes::default(); + for value in types { + match value { + FileType::File => file_types.files = true, + FileType::Directory => file_types.directories = true, + FileType::Symlink => file_types.symlinks = true, + FileType::Executable => { + file_types.executables_only = true; + file_types.files = true; + } + FileType::Empty => file_types.empty_only = true, + FileType::BlockDevice => file_types.block_devices = true, + FileType::CharDevice => file_types.char_devices = true, + FileType::Socket => file_types.sockets = true, + FileType::Pipe => file_types.pipes = true, + } + } + + // If only 'empty' was specified, search for both files and directories: + if file_types.empty_only && !(file_types.files || file_types.directories) { + file_types.files = true; + file_types.directories = true; + } + + file_types + } + + pub fn should_ignore(&self, entry: &dir_entry::DirEntry) -> bool { + if let Some(ref entry_type) = entry.file_type() { + (!self.files && entry_type.is_file()) + || (!self.directories && entry_type.is_dir()) + || (!self.symlinks && entry_type.is_symlink()) + || (!self.block_devices && filesystem::is_block_device(*entry_type)) + || (!self.char_devices && filesystem::is_char_device(*entry_type)) + || (!self.sockets && filesystem::is_socket(*entry_type)) + || (!self.pipes && filesystem::is_pipe(*entry_type)) + || (self.executables_only && !entry.path().executable()) + || (self.empty_only && !filesystem::is_empty(entry)) + || !(entry_type.is_file() + || entry_type.is_dir() + || entry_type.is_symlink() + || filesystem::is_block_device(*entry_type) + || filesystem::is_char_device(*entry_type) + || filesystem::is_socket(*entry_type) + || filesystem::is_pipe(*entry_type)) + } else { + true + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn empty_only_implies_files_and_directories() { + let ft = FileTypes::from_types(&[FileType::Empty]); + assert!(ft.empty_only); + assert!(ft.files); + assert!(ft.directories); + } + + #[test] + fn empty_with_explicit_file_keeps_directories_off() { + let ft = FileTypes::from_types(&[FileType::Empty, FileType::File]); + assert!(ft.empty_only); + assert!(ft.files); + assert!(!ft.directories); + } + + #[test] + fn executable_implies_files() { + let ft = FileTypes::from_types(&[FileType::Executable]); + assert!(ft.executables_only); + assert!(ft.files); + assert!(!ft.directories); + } + + #[test] + fn union_of_multiple_types() { + let ft = FileTypes::from_types(&[FileType::File, FileType::Symlink]); + assert!(ft.files); + assert!(ft.symlinks); + assert!(!ft.directories); + assert!(!ft.executables_only); + assert!(!ft.empty_only); + } +} diff --git a/userland/fd/src/filter/mod.rs b/userland/fd/src/filter/mod.rs new file mode 100644 index 0000000..5e45d3b --- /dev/null +++ b/userland/fd/src/filter/mod.rs @@ -0,0 +1,11 @@ +pub use self::size::SizeFilter; +pub use self::time::TimeFilter; + +#[cfg(unix)] +pub use self::owner::OwnerFilter; + +mod size; +mod time; + +#[cfg(unix)] +mod owner; diff --git a/userland/fd/src/filter/owner.rs b/userland/fd/src/filter/owner.rs new file mode 100644 index 0000000..66b7f9d --- /dev/null +++ b/userland/fd/src/filter/owner.rs @@ -0,0 +1,140 @@ +use anyhow::{Result, anyhow}; +use nix::unistd::{Group, User}; +use std::fs; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct OwnerFilter { + uid: Check, + gid: Check, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Check { + Equal(T), + NotEq(T), + Ignore, +} + +impl OwnerFilter { + const IGNORE: Self = Self { + uid: Check::Ignore, + gid: Check::Ignore, + }; + + /// Parses an owner constraint + /// Returns an error if the string is invalid + /// Returns Ok(None) when string is acceptable but a noop (such as "" or ":") + pub fn from_string(input: &str) -> Result { + let mut it = input.split(':'); + let (fst, snd) = (it.next(), it.next()); + + if it.next().is_some() { + return Err(anyhow!( + "more than one ':' present in owner string '{}'. See 'fd --help'.", + input + )); + } + + let uid = Check::parse(fst, |s| { + if let Ok(uid) = s.parse() { + Ok(uid) + } else { + User::from_name(s)? + .map(|user| user.uid.as_raw()) + .ok_or_else(|| anyhow!("'{}' is not a recognized user name", s)) + } + })?; + let gid = Check::parse(snd, |s| { + if let Ok(gid) = s.parse() { + Ok(gid) + } else { + Group::from_name(s)? + .map(|group| group.gid.as_raw()) + .ok_or_else(|| anyhow!("'{}' is not a recognized group name", s)) + } + })?; + + Ok(Self { uid, gid }) + } + + /// If self is a no-op (ignore both uid and gid) then return `None`, otherwise wrap in a `Some` + pub fn filter_ignore(self) -> Option { + if self == Self::IGNORE { + None + } else { + Some(self) + } + } + + pub fn matches(&self, md: &fs::Metadata) -> bool { + use std::os::unix::fs::MetadataExt; + + self.uid.check(md.uid()) && self.gid.check(md.gid()) + } +} + +impl Check { + fn check(&self, v: T) -> bool { + match self { + Self::Equal(x) => v == *x, + Self::NotEq(x) => v != *x, + Self::Ignore => true, + } + } + + fn parse(s: Option<&str>, f: F) -> Result + where + F: Fn(&str) -> Result, + { + let (s, equality) = match s { + Some("") | None => return Ok(Self::Ignore), + Some(s) if s.starts_with('!') => (&s[1..], false), + Some(s) => (s, true), + }; + + f(s).map(|x| { + if equality { + Self::Equal(x) + } else { + Self::NotEq(x) + } + }) + } +} + +#[cfg(test)] +mod owner_parsing { + use super::OwnerFilter; + + macro_rules! owner_tests { + ($($name:ident: $value:expr => $result:pat,)*) => { + $( + #[test] + fn $name() { + let o = OwnerFilter::from_string($value); + match o { + $result => {}, + _ => panic!("{:?} does not match {}", o, stringify!($result)), + } + } + )* + }; + } + + use super::Check::*; + owner_tests! { + empty: "" => Ok(OwnerFilter::IGNORE), + uid_only: "5" => Ok(OwnerFilter { uid: Equal(5), gid: Ignore }), + uid_gid: "9:3" => Ok(OwnerFilter { uid: Equal(9), gid: Equal(3) }), + gid_only: ":8" => Ok(OwnerFilter { uid: Ignore, gid: Equal(8) }), + colon_only: ":" => Ok(OwnerFilter::IGNORE), + trailing: "5:" => Ok(OwnerFilter { uid: Equal(5), gid: Ignore }), + + uid_negate: "!5" => Ok(OwnerFilter { uid: NotEq(5), gid: Ignore }), + both_negate:"!4:!3" => Ok(OwnerFilter { uid: NotEq(4), gid: NotEq(3) }), + uid_not_gid:"6:!8" => Ok(OwnerFilter { uid: Equal(6), gid: NotEq(8) }), + + more_colons:"3:5:" => Err(_), + only_colons:"::" => Err(_), + } +} diff --git a/userland/fd/src/filter/size.rs b/userland/fd/src/filter/size.rs new file mode 100644 index 0000000..740ca0c --- /dev/null +++ b/userland/fd/src/filter/size.rs @@ -0,0 +1,258 @@ +use std::sync::OnceLock; + +use anyhow::anyhow; +use regex::Regex; + +static SIZE_CAPTURES: OnceLock = OnceLock::new(); + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum SizeFilter { + Max(u64), + Min(u64), + Equals(u64), +} + +// SI prefixes (powers of 10) +const KILO: u64 = 1000; +const MEGA: u64 = KILO * 1000; +const GIGA: u64 = MEGA * 1000; +const TERA: u64 = GIGA * 1000; + +// Binary prefixes (powers of 2) +const KIBI: u64 = 1024; +const MEBI: u64 = KIBI * 1024; +const GIBI: u64 = MEBI * 1024; +const TEBI: u64 = GIBI * 1024; + +impl SizeFilter { + pub fn from_string(s: &str) -> anyhow::Result { + Self::parse_opt(s) + .ok_or_else(|| anyhow!("'{}' is not a valid size constraint. See 'fd --help'.", s)) + } + + fn parse_opt(s: &str) -> Option { + let pattern = + SIZE_CAPTURES.get_or_init(|| Regex::new(r"(?i)^([+-]?)(\d+)(b|[kmgt]i?b?)$").unwrap()); + if !pattern.is_match(s) { + return None; + } + + let captures = pattern.captures(s)?; + let limit_kind = captures.get(1).map_or("+", |m| m.as_str()); + let quantity = captures + .get(2) + .and_then(|v| v.as_str().parse::().ok())?; + + let multiplier = match &captures.get(3).map_or("b", |m| m.as_str()).to_lowercase()[..] { + v if v.starts_with("ki") => KIBI, + v if v.starts_with('k') => KILO, + v if v.starts_with("mi") => MEBI, + v if v.starts_with('m') => MEGA, + v if v.starts_with("gi") => GIBI, + v if v.starts_with('g') => GIGA, + v if v.starts_with("ti") => TEBI, + v if v.starts_with('t') => TERA, + "b" => 1, + _ => return None, + }; + + let size = quantity.checked_mul(multiplier)?; + + match limit_kind { + "+" => Some(Self::Min(size)), + "-" => Some(Self::Max(size)), + "" => Some(Self::Equals(size)), + _ => None, + } + } + + pub fn is_within(&self, size: u64) -> bool { + match *self { + Self::Max(limit) => size <= limit, + Self::Min(limit) => size >= limit, + Self::Equals(limit) => size == limit, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + macro_rules! gen_size_filter_parse_test { + ($($name: ident: $val: expr,)*) => { + $( + #[test] + fn $name() { + let (txt, expected) = $val; + let actual = SizeFilter::from_string(txt).unwrap(); + assert_eq!(actual, expected); + } + )* + }; + } + + // Parsing and size conversion tests data. Ensure that each type gets properly interpreted. + // Call with higher base values to ensure expected multiplication (only need a couple) + gen_size_filter_parse_test! { + byte_plus: ("+1b", SizeFilter::Min(1)), + byte_plus_multiplier: ("+10b", SizeFilter::Min(10)), + byte_minus: ("-1b", SizeFilter::Max(1)), + kilo_plus: ("+1k", SizeFilter::Min(1000)), + kilo_plus_suffix: ("+1kb", SizeFilter::Min(1000)), + kilo_minus: ("-1k", SizeFilter::Max(1000)), + kilo_minus_multiplier: ("-100k", SizeFilter::Max(100_000)), + kilo_minus_suffix: ("-1kb", SizeFilter::Max(1000)), + kilo_plus_upper: ("+1K", SizeFilter::Min(1000)), + kilo_plus_suffix_upper: ("+1KB", SizeFilter::Min(1000)), + kilo_minus_upper: ("-1K", SizeFilter::Max(1000)), + kilo_minus_suffix_upper: ("-1Kb", SizeFilter::Max(1000)), + kibi_plus: ("+1ki", SizeFilter::Min(1024)), + kibi_plus_multiplier: ("+10ki", SizeFilter::Min(10_240)), + kibi_plus_suffix: ("+1kib", SizeFilter::Min(1024)), + kibi_minus: ("-1ki", SizeFilter::Max(1024)), + kibi_minus_multiplier: ("-100ki", SizeFilter::Max(102_400)), + kibi_minus_suffix: ("-1kib", SizeFilter::Max(1024)), + kibi_plus_upper: ("+1KI", SizeFilter::Min(1024)), + kibi_plus_suffix_upper: ("+1KiB", SizeFilter::Min(1024)), + kibi_minus_upper: ("-1Ki", SizeFilter::Max(1024)), + kibi_minus_suffix_upper: ("-1KIB", SizeFilter::Max(1024)), + mega_plus: ("+1m", SizeFilter::Min(1_000_000)), + mega_plus_suffix: ("+1mb", SizeFilter::Min(1_000_000)), + mega_minus: ("-1m", SizeFilter::Max(1_000_000)), + mega_minus_suffix: ("-1mb", SizeFilter::Max(1_000_000)), + mega_plus_upper: ("+1M", SizeFilter::Min(1_000_000)), + mega_plus_suffix_upper: ("+1MB", SizeFilter::Min(1_000_000)), + mega_minus_upper: ("-1M", SizeFilter::Max(1_000_000)), + mega_minus_suffix_upper: ("-1Mb", SizeFilter::Max(1_000_000)), + mebi_plus: ("+1mi", SizeFilter::Min(1_048_576)), + mebi_plus_suffix: ("+1mib", SizeFilter::Min(1_048_576)), + mebi_minus: ("-1mi", SizeFilter::Max(1_048_576)), + mebi_minus_suffix: ("-1mib", SizeFilter::Max(1_048_576)), + mebi_plus_upper: ("+1MI", SizeFilter::Min(1_048_576)), + mebi_plus_suffix_upper: ("+1MiB", SizeFilter::Min(1_048_576)), + mebi_minus_upper: ("-1Mi", SizeFilter::Max(1_048_576)), + mebi_minus_suffix_upper: ("-1MIB", SizeFilter::Max(1_048_576)), + giga_plus: ("+1g", SizeFilter::Min(1_000_000_000)), + giga_plus_suffix: ("+1gb", SizeFilter::Min(1_000_000_000)), + giga_minus: ("-1g", SizeFilter::Max(1_000_000_000)), + giga_minus_suffix: ("-1gb", SizeFilter::Max(1_000_000_000)), + giga_plus_upper: ("+1G", SizeFilter::Min(1_000_000_000)), + giga_plus_suffix_upper: ("+1GB", SizeFilter::Min(1_000_000_000)), + giga_minus_upper: ("-1G", SizeFilter::Max(1_000_000_000)), + giga_minus_suffix_upper: ("-1Gb", SizeFilter::Max(1_000_000_000)), + gibi_plus: ("+1gi", SizeFilter::Min(1_073_741_824)), + gibi_plus_suffix: ("+1gib", SizeFilter::Min(1_073_741_824)), + gibi_minus: ("-1gi", SizeFilter::Max(1_073_741_824)), + gibi_minus_suffix: ("-1gib", SizeFilter::Max(1_073_741_824)), + gibi_plus_upper: ("+1GI", SizeFilter::Min(1_073_741_824)), + gibi_plus_suffix_upper: ("+1GiB", SizeFilter::Min(1_073_741_824)), + gibi_minus_upper: ("-1Gi", SizeFilter::Max(1_073_741_824)), + gibi_minus_suffix_upper: ("-1GIB", SizeFilter::Max(1_073_741_824)), + tera_plus: ("+1t", SizeFilter::Min(1_000_000_000_000)), + tera_plus_suffix: ("+1tb", SizeFilter::Min(1_000_000_000_000)), + tera_minus: ("-1t", SizeFilter::Max(1_000_000_000_000)), + tera_minus_suffix: ("-1tb", SizeFilter::Max(1_000_000_000_000)), + tera_plus_upper: ("+1T", SizeFilter::Min(1_000_000_000_000)), + tera_plus_suffix_upper: ("+1TB", SizeFilter::Min(1_000_000_000_000)), + tera_minus_upper: ("-1T", SizeFilter::Max(1_000_000_000_000)), + tera_minus_suffix_upper: ("-1Tb", SizeFilter::Max(1_000_000_000_000)), + tebi_plus: ("+1ti", SizeFilter::Min(1_099_511_627_776)), + tebi_plus_suffix: ("+1tib", SizeFilter::Min(1_099_511_627_776)), + tebi_minus: ("-1ti", SizeFilter::Max(1_099_511_627_776)), + tebi_minus_suffix: ("-1tib", SizeFilter::Max(1_099_511_627_776)), + tebi_plus_upper: ("+1TI", SizeFilter::Min(1_099_511_627_776)), + tebi_plus_suffix_upper: ("+1TiB", SizeFilter::Min(1_099_511_627_776)), + tebi_minus_upper: ("-1Ti", SizeFilter::Max(1_099_511_627_776)), + tebi_minus_suffix_upper: ("-1TIB", SizeFilter::Max(1_099_511_627_776)), + } + + /// Invalid parse testing + macro_rules! gen_size_filter_failure { + ($($name:ident: $value:expr,)*) => { + $( + #[test] + fn $name() { + let i = SizeFilter::from_string($value); + assert!(i.is_err()); + } + )* + }; + } + + // Invalid parse data + gen_size_filter_failure! { + ensure_missing_number_returns_none: "+g", + ensure_missing_unit_returns_none: "+18", + ensure_bad_format_returns_none_1: "$10M", + ensure_bad_format_returns_none_2: "badval", + ensure_bad_format_returns_none_3: "9999", + ensure_invalid_unit_returns_none_1: "+50a", + ensure_invalid_unit_returns_none_2: "-10v", + ensure_invalid_unit_returns_none_3: "+1Mv", + ensure_bib_format_returns_none: "+1bib", + ensure_bb_format_returns_none: "+1bb", + } + + #[test] + fn is_within_less_than() { + let f = SizeFilter::from_string("-1k").unwrap(); + assert!(f.is_within(999)); + } + + #[test] + fn is_within_less_than_equal() { + let f = SizeFilter::from_string("-1k").unwrap(); + assert!(f.is_within(1000)); + } + + #[test] + fn is_within_greater_than() { + let f = SizeFilter::from_string("+1k").unwrap(); + assert!(f.is_within(1001)); + } + + #[test] + fn is_within_greater_than_equal() { + let f = SizeFilter::from_string("+1K").unwrap(); + assert!(f.is_within(1000)); + } + + #[test] + fn ensure_size_multiplication_overflow_returns_error() { + let invalid = [ + "+18446744073709551615k", + "-18446744073709551615m", + "18446744073709551615g", + "+18446744073709551615ti", + "-18446744073709551615tib", + ]; + + for input in invalid { + assert!( + SizeFilter::from_string(input).is_err(), + "Expected {input:?} to fail due to overflow" + ); + } + } + + #[test] + fn ensure_size_multiplication_boundary() { + let max_valid = u64::MAX / KILO; + + assert_eq!( + SizeFilter::from_string(&format!("{max_valid}k")).unwrap(), + SizeFilter::Equals(max_valid * KILO) + ); + + assert!(SizeFilter::from_string(&format!("{}k", max_valid + 1)).is_err()); + } + + #[test] + fn ensure_max_u64_does_not_overflow() { + assert_eq!( + SizeFilter::from_string("18446744073709551615b").unwrap(), + SizeFilter::Equals(u64::MAX) + ); + } +} diff --git a/userland/fd/src/filter/time.rs b/userland/fd/src/filter/time.rs new file mode 100644 index 0000000..248b5a1 --- /dev/null +++ b/userland/fd/src/filter/time.rs @@ -0,0 +1,210 @@ +use jiff::{Span, Timestamp, Zoned, civil::DateTime, tz::TimeZone}; + +use std::time::{Duration, SystemTime, UNIX_EPOCH}; + +/// Filter based on time ranges. +#[derive(Debug, PartialEq, Eq)] +pub enum TimeFilter { + Before(SystemTime), + After(SystemTime), +} + +#[cfg(not(test))] +fn now() -> Zoned { + Zoned::now() +} + +#[cfg(test)] +thread_local! { + static TESTTIME: std::cell::RefCell> = None.into(); +} + +/// This allows us to set a specific time when running tests +#[cfg(test)] +fn now() -> Zoned { + TESTTIME.with_borrow(|reftime| reftime.as_ref().cloned().unwrap_or_else(Zoned::now)) +} + +impl TimeFilter { + fn from_str(s: &str) -> Option { + if let Ok(span) = s.parse::() { + let datetime = now().checked_sub(span).ok()?; + Some(datetime.into()) + } else if let Ok(timestamp) = s.parse::() { + Some(timestamp.into()) + } else if let Ok(datetime) = s.parse::() { + Some( + TimeZone::system() + .to_ambiguous_zoned(datetime) + .later() + .ok()? + .into(), + ) + } else { + let timestamp_secs: u64 = s.strip_prefix('@')?.parse().ok()?; + UNIX_EPOCH.checked_add(Duration::from_secs(timestamp_secs)) + } + } + + pub fn before(s: &str) -> Option { + Self::from_str(s).map(TimeFilter::Before) + } + + pub fn after(s: &str) -> Option { + Self::from_str(s).map(TimeFilter::After) + } + + pub fn applies_to(&self, t: &SystemTime) -> bool { + match self { + Self::Before(limit) => t < limit, + Self::After(limit) => t > limit, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::time::Duration; + + struct TestTime(SystemTime); + + impl TestTime { + fn new(time: Zoned) -> Self { + TESTTIME.with_borrow_mut(|t| *t = Some(time.clone())); + TestTime(time.into()) + } + + fn set(&mut self, time: Zoned) { + TESTTIME.with_borrow_mut(|t| *t = Some(time.clone())); + self.0 = time.into(); + } + + fn timestamp(&self) -> SystemTime { + self.0 + } + } + + impl Drop for TestTime { + fn drop(&mut self) { + // Stop using manually set times + TESTTIME.with_borrow_mut(|t| *t = None); + } + } + + #[test] + fn is_time_filter_applicable() { + let local_tz = TimeZone::system(); + let mut test_time = TestTime::new( + local_tz + .to_ambiguous_zoned("2010-10-10 10:10:10".parse::().unwrap()) + .later() + .unwrap(), + ); + let mut ref_time = test_time.timestamp(); + + assert!(TimeFilter::after("1min").unwrap().applies_to(&ref_time)); + assert!(!TimeFilter::before("1min").unwrap().applies_to(&ref_time)); + + let t1m_ago = ref_time - Duration::from_secs(60); + assert!(!TimeFilter::after("30sec").unwrap().applies_to(&t1m_ago)); + assert!(TimeFilter::after("2min").unwrap().applies_to(&t1m_ago)); + + assert!(TimeFilter::before("30sec").unwrap().applies_to(&t1m_ago)); + assert!(!TimeFilter::before("2min").unwrap().applies_to(&t1m_ago)); + + let t10s_before = "2010-10-10 10:10:00"; + assert!( + !TimeFilter::before(t10s_before) + .unwrap() + .applies_to(&ref_time) + ); + assert!( + TimeFilter::before(t10s_before) + .unwrap() + .applies_to(&t1m_ago) + ); + + assert!( + TimeFilter::after(t10s_before) + .unwrap() + .applies_to(&ref_time) + ); + assert!(!TimeFilter::after(t10s_before).unwrap().applies_to(&t1m_ago)); + + let same_day = "2010-10-10"; + assert!(!TimeFilter::before(same_day).unwrap().applies_to(&ref_time)); + assert!(!TimeFilter::before(same_day).unwrap().applies_to(&t1m_ago)); + + assert!(TimeFilter::after(same_day).unwrap().applies_to(&ref_time)); + assert!(TimeFilter::after(same_day).unwrap().applies_to(&t1m_ago)); + + test_time.set( + "2010-10-10T10:10:10+00:00" + .parse::() + .unwrap() + .to_zoned(local_tz.clone()), + ); + ref_time = test_time.timestamp(); + let t1m_ago = ref_time - Duration::from_secs(60); + let t10s_before = "2010-10-10T10:10:00+00:00"; + assert!( + !TimeFilter::before(t10s_before) + .unwrap() + .applies_to(&ref_time) + ); + assert!( + TimeFilter::before(t10s_before) + .unwrap() + .applies_to(&t1m_ago) + ); + + assert!( + TimeFilter::after(t10s_before) + .unwrap() + .applies_to(&ref_time) + ); + assert!(!TimeFilter::after(t10s_before).unwrap().applies_to(&t1m_ago)); + + let ref_timestamp = 1707723412u64; // Mon Feb 12 07:36:52 UTC 2024 + test_time.set( + "2024-02-12T07:36:52+00:00" + .parse::() + .unwrap() + .to_zoned(local_tz), + ); + ref_time = test_time.timestamp(); + let t1m_ago = ref_time - Duration::from_secs(60); + let t1s_later = ref_time + Duration::from_secs(1); + // Timestamp only supported via '@' prefix + assert!(TimeFilter::before(&ref_timestamp.to_string()).is_none()); + assert!( + TimeFilter::before(&format!("@{ref_timestamp}")) + .unwrap() + .applies_to(&t1m_ago) + ); + assert!( + !TimeFilter::before(&format!("@{ref_timestamp}")) + .unwrap() + .applies_to(&t1s_later) + ); + assert!( + !TimeFilter::after(&format!("@{ref_timestamp}")) + .unwrap() + .applies_to(&t1m_ago) + ); + assert!( + TimeFilter::after(&format!("@{ref_timestamp}")) + .unwrap() + .applies_to(&t1s_later) + ); + } + + #[test] + fn out_of_range_unix_timestamp_is_rejected() { + // A '@' timestamp large enough to overflow SystemTime must return + // None rather than panicking. + assert!(TimeFilter::before(&format!("@{}", u64::MAX)).is_none()); + assert!(TimeFilter::after(&format!("@{}", u64::MAX)).is_none()); + } +} diff --git a/userland/fd/src/fmt/input.rs b/userland/fd/src/fmt/input.rs new file mode 100644 index 0000000..a599432 --- /dev/null +++ b/userland/fd/src/fmt/input.rs @@ -0,0 +1,87 @@ +use std::ffi::{OsStr, OsString}; +use std::path::{Path, PathBuf}; + +use crate::filesystem::strip_current_dir; + +/// Removes the parent component of the path +pub fn basename(path: &Path) -> &OsStr { + path.file_name().unwrap_or(path.as_os_str()) +} + +/// Removes the extension from the path +pub fn remove_extension(path: &Path) -> OsString { + let dirname = dirname(path); + let stem = path.file_stem().unwrap_or(path.as_os_str()); + + let path = PathBuf::from(dirname).join(stem); + + strip_current_dir(&path).to_owned().into_os_string() +} + +/// Removes the basename from the path. +pub fn dirname(path: &Path) -> OsString { + path.parent() + .map(|p| { + if p == OsStr::new("") { + OsString::from(".") + } else { + p.as_os_str().to_owned() + } + }) + .unwrap_or_else(|| path.as_os_str().to_owned()) +} + +#[cfg(test)] +mod path_tests { + use super::*; + use std::path::MAIN_SEPARATOR_STR; + + fn correct(input: &str) -> String { + input.replace('/', MAIN_SEPARATOR_STR) + } + + macro_rules! func_tests { + ($($name:ident: $func:ident for $input:expr => $output:expr)+) => { + $( + #[test] + fn $name() { + let input_path = PathBuf::from(&correct($input)); + let output_string = OsString::from(correct($output)); + assert_eq!($func(&input_path), output_string); + } + )+ + } + } + + func_tests! { + remove_ext_simple: remove_extension for "foo.txt" => "foo" + remove_ext_dir: remove_extension for "dir/foo.txt" => "dir/foo" + hidden: remove_extension for ".foo" => ".foo" + remove_ext_utf8: remove_extension for "💖.txt" => "💖" + remove_ext_empty: remove_extension for "" => "" + + basename_simple: basename for "foo.txt" => "foo.txt" + basename_dir: basename for "dir/foo.txt" => "foo.txt" + basename_empty: basename for "" => "" + basename_utf8_0: basename for "💖/foo.txt" => "foo.txt" + basename_utf8_1: basename for "dir/💖.txt" => "💖.txt" + + dirname_simple: dirname for "foo.txt" => "." + dirname_dir: dirname for "dir/foo.txt" => "dir" + dirname_utf8_0: dirname for "💖/foo.txt" => "💖" + dirname_utf8_1: dirname for "dir/💖.txt" => "dir" + } + + #[test] + #[cfg(windows)] + fn dirname_root() { + assert_eq!(dirname(&PathBuf::from("C:")), OsString::from("C:")); + assert_eq!(dirname(&PathBuf::from("\\")), OsString::from("\\")); + } + + #[test] + #[cfg(not(windows))] + fn dirname_root() { + assert_eq!(dirname(&PathBuf::from("/")), OsString::from("/")); + } +} diff --git a/userland/fd/src/fmt/mod.rs b/userland/fd/src/fmt/mod.rs new file mode 100644 index 0000000..79eb704 --- /dev/null +++ b/userland/fd/src/fmt/mod.rs @@ -0,0 +1,281 @@ +mod input; + +use std::borrow::Cow; +use std::ffi::{OsStr, OsString}; +use std::fmt::{self, Display, Formatter}; +use std::path::{Component, Path, Prefix}; +use std::sync::OnceLock; + +use aho_corasick::AhoCorasick; + +use self::input::{basename, dirname, remove_extension}; + +/// Designates what should be written to a buffer +/// +/// Each `Token` contains either text, or a placeholder variant, which will be used to generate +/// commands after all tokens for a given command template have been collected. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum Token { + Placeholder, + Basename, + Parent, + NoExt, + BasenameNoExt, + Text(String), +} + +impl Display for Token { + fn fmt(&self, f: &mut Formatter) -> fmt::Result { + match *self { + Self::Placeholder => f.write_str("{}")?, + Self::Basename => f.write_str("{/}")?, + Self::Parent => f.write_str("{//}")?, + Self::NoExt => f.write_str("{.}")?, + Self::BasenameNoExt => f.write_str("{/.}")?, + Self::Text(ref string) => f.write_str(string)?, + } + Ok(()) + } +} + +/// A parsed format string +/// +/// This is either a collection of `Token`s including at least one placeholder variant, +/// or a fixed text. +#[derive(Clone, Debug, PartialEq)] +pub enum FormatTemplate { + Tokens(Vec), + Text(String), +} + +static PLACEHOLDERS: OnceLock = OnceLock::new(); + +impl FormatTemplate { + pub fn has_tokens(&self) -> bool { + matches!(self, Self::Tokens(_)) + } + + pub fn parse(fmt: &str) -> Self { + // NOTE: we assume that { and } have the same length + const BRACE_LEN: usize = '{'.len_utf8(); + let mut tokens = Vec::new(); + let mut remaining = fmt; + let mut buf = String::new(); + let placeholders = PLACEHOLDERS.get_or_init(|| { + AhoCorasick::new(["{{", "}}", "{}", "{/}", "{//}", "{.}", "{/.}"]).unwrap() + }); + while let Some(m) = placeholders.find(remaining) { + match m.pattern().as_u32() { + 0 | 1 => { + // we found an escaped {{ or }}, so add + // everything up to the first char to the buffer + // then skip the second one. + buf += &remaining[..m.start() + BRACE_LEN]; + remaining = &remaining[m.end()..]; + } + id if !remaining[m.end()..].starts_with('}') => { + buf += &remaining[..m.start()]; + if !buf.is_empty() { + tokens.push(Token::Text(std::mem::take(&mut buf))); + } + tokens.push(token_from_pattern_id(id)); + remaining = &remaining[m.end()..]; + } + _ => { + // We got a normal pattern, but the final "}" + // is escaped, so add up to that to the buffer, then + // skip the final } + buf += &remaining[..m.end()]; + remaining = &remaining[m.end() + BRACE_LEN..]; + } + } + } + // Add the rest of the string to the buffer, and add the final buffer to the tokens + if !remaining.is_empty() { + buf += remaining; + } + if tokens.is_empty() { + // No placeholders were found, so just return the text + return Self::Text(buf); + } + // Add final text segment + if !buf.is_empty() { + tokens.push(Token::Text(buf)); + } + debug_assert!(!tokens.is_empty()); + Self::Tokens(tokens) + } + + /// Generate a result string from this template. If path_separator is Some, then it will replace + /// the path separator in all placeholder tokens. Fixed text and tokens are not affected by + /// path separator substitution. + pub fn generate(&self, path: impl AsRef, path_separator: Option<&str>) -> OsString { + use Token::*; + let path = path.as_ref(); + + match *self { + Self::Tokens(ref tokens) => { + let mut s = OsString::new(); + for token in tokens { + match token { + Basename => s.push(Self::replace_separator(basename(path), path_separator)), + BasenameNoExt => s.push(Self::replace_separator( + &remove_extension(basename(path).as_ref()), + path_separator, + )), + NoExt => s.push(Self::replace_separator( + &remove_extension(path), + path_separator, + )), + Parent => s.push(Self::replace_separator(&dirname(path), path_separator)), + Placeholder => { + s.push(Self::replace_separator(path.as_ref(), path_separator)) + } + Text(string) => s.push(string), + } + } + s + } + Self::Text(ref text) => OsString::from(text), + } + } + + /// Replace the path separator in the input with the custom separator string. If path_separator + /// is None, simply return a borrowed Cow of the input. Otherwise, the input is + /// interpreted as a Path and its components are iterated through and re-joined into a new + /// OsString. + fn replace_separator<'a>(path: &'a OsStr, path_separator: Option<&str>) -> Cow<'a, OsStr> { + // fast-path - no replacement necessary + if path_separator.is_none() { + return Cow::Borrowed(path); + } + + let path_separator = path_separator.unwrap(); + let mut out = OsString::with_capacity(path.len()); + let mut components = Path::new(path).components().peekable(); + + while let Some(comp) = components.next() { + match comp { + // Absolute paths on Windows are tricky. A Prefix component is usually a drive + // letter or UNC path, and is usually followed by RootDir. There are also + // "verbatim" prefixes beginning with "\\?\" that skip normalization. We choose to + // ignore verbatim path prefixes here because they're very rare, might be + // impossible to reach here, and there's no good way to deal with them. If users + // are doing something advanced involving verbatim windows paths, they can do their + // own output filtering with a tool like sed. + Component::Prefix(prefix) => { + if let Prefix::UNC(server, share) = prefix.kind() { + // Prefix::UNC is a parsed version of '\\server\share' + out.push(path_separator); + out.push(path_separator); + out.push(server); + out.push(path_separator); + out.push(share); + } else { + // All other Windows prefix types are rendered as-is. This results in e.g. "C:" for + // drive letters. DeviceNS and Verbatim* prefixes won't have backslashes converted, + // but they're not returned by directories fd can search anyway so we don't worry + // about them. + out.push(comp.as_os_str()); + } + } + + // Root directory is always replaced with the custom separator. + Component::RootDir => out.push(path_separator), + + // Everything else is joined normally, with a trailing separator if we're not last + _ => { + out.push(comp.as_os_str()); + if components.peek().is_some() { + out.push(path_separator); + } + } + } + } + Cow::Owned(out) + } +} + +// Convert the id from an aho-corasick match to the +// appropriate token +fn token_from_pattern_id(id: u32) -> Token { + use Token::*; + match id { + 2 => Placeholder, + 3 => Basename, + 4 => Parent, + 5 => NoExt, + 6 => BasenameNoExt, + _ => unreachable!(), + } +} + +#[cfg(test)] +mod fmt_tests { + use super::*; + use std::path::PathBuf; + + #[test] + fn parse_no_placeholders() { + let templ = FormatTemplate::parse("This string has no placeholders"); + assert_eq!( + templ, + FormatTemplate::Text("This string has no placeholders".into()) + ); + } + + #[test] + fn parse_only_brace_escapes() { + let templ = FormatTemplate::parse("This string only has escapes like {{ and }}"); + assert_eq!( + templ, + FormatTemplate::Text("This string only has escapes like { and }".into()) + ); + } + + #[test] + fn all_placeholders() { + use Token::*; + + let templ = FormatTemplate::parse( + "{{path={} \ + basename={/} \ + parent={//} \ + noExt={.} \ + basenameNoExt={/.} \ + }}", + ); + assert_eq!( + templ, + FormatTemplate::Tokens(vec![ + Text("{path=".into()), + Placeholder, + Text(" basename=".into()), + Basename, + Text(" parent=".into()), + Parent, + Text(" noExt=".into()), + NoExt, + Text(" basenameNoExt=".into()), + BasenameNoExt, + Text(" }".into()), + ]) + ); + + let mut path = PathBuf::new(); + path.push("a"); + path.push("folder"); + path.push("file.txt"); + + let expanded = templ.generate(&path, Some("/")).into_string().unwrap(); + + assert_eq!( + expanded, + "{path=a/folder/file.txt \ + basename=file.txt \ + parent=a/folder \ + noExt=a/folder/file \ + basenameNoExt=file }" + ); + } +} diff --git a/userland/fd/src/hyperlink.rs b/userland/fd/src/hyperlink.rs new file mode 100644 index 0000000..0973e4e --- /dev/null +++ b/userland/fd/src/hyperlink.rs @@ -0,0 +1,87 @@ +use crate::filesystem::absolute_path; +use std::fmt::{self, Formatter, Write}; +use std::path::{Path, PathBuf}; + +pub(crate) struct PathUrl(PathBuf); + +impl PathUrl { + pub(crate) fn new(path: &Path) -> Option { + Some(Self(absolute_path(path).ok()?)) + } +} + +impl fmt::Display for PathUrl { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + write!(f, "file://{}", host())?; + let bytes = self.0.as_os_str().as_encoded_bytes(); + for &byte in bytes.iter() { + encode(f, byte)?; + } + Ok(()) + } +} + +fn encode(f: &mut Formatter, byte: u8) -> fmt::Result { + // NOTE: + // Most terminals can handle non-ascii unicode characters in a file url fine. But on some OSes (notably + // windows), the encoded bytes of the path may not be valid UTF-8. Since we don't know if a + // byte >= 128 is part of a valid UTF-8 encoding or not, we just percent encode any non-ascii + // byte. + // Percent encoding these bytes is probably safer anyway. + match byte { + b'0'..=b'9' | b'A'..=b'Z' | b'a'..=b'z' | b'/' | b':' | b'-' | b'.' | b'_' | b'~' => { + f.write_char(byte.into()) + } + #[cfg(windows)] + b'\\' => f.write_char('/'), + _ => { + write!(f, "%{byte:02X}") + } + } +} + +#[cfg(unix)] +fn host() -> &'static str { + use std::sync::OnceLock; + + static HOSTNAME: OnceLock = OnceLock::new(); + + HOSTNAME + .get_or_init(|| { + nix::unistd::gethostname() + .ok() + .and_then(|h| h.into_string().ok()) + .unwrap_or_default() + }) + .as_ref() +} + +#[cfg(not(unix))] +const fn host() -> &'static str { + "/" +} + +#[cfg(test)] +mod test { + use super::*; + + // This allows us to test the encoding without having to worry about the host, or absolute path + struct Encoded(&'static str); + + impl fmt::Display for Encoded { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + for byte in self.0.bytes() { + encode(f, byte)?; + } + Ok(()) + } + } + + #[test] + fn test_unicode_encoding() { + assert_eq!( + Encoded("$*\x1bßé/∫😃\x07").to_string(), + "%24%2A%1B%C3%9F%C3%A9/%E2%88%AB%F0%9F%98%83%07", + ); + } +} diff --git a/userland/fd/src/main.rs b/userland/fd/src/main.rs new file mode 100644 index 0000000..a747090 --- /dev/null +++ b/userland/fd/src/main.rs @@ -0,0 +1,515 @@ +// needs to be first because it defines a macro +#[macro_use] +mod error; + +mod cli; +mod config; +mod dir_entry; +mod exec; +mod exit_codes; +mod filesystem; +mod filetypes; +mod filter; +mod fmt; +mod hyperlink; +mod output; +mod patterns; +mod regex_helper; +mod sanitize; +mod walk; +#[cfg(target_os = "wasi")] +mod wasi_walk; + +use std::env; +use std::io::IsTerminal; +#[cfg(any(windows, feature = "completions"))] +use std::path::Path; +use std::sync::Arc; + +use anyhow::{Context, Result, anyhow, bail}; +use clap::Parser; +#[cfg(feature = "completions")] +use clap::CommandFactory; +use lscolors::LsColors; +use patterns::PatternType; +use regex::bytes::RegexSetBuilder; + +use crate::cli::{ColorWhen, HyperlinkWhen, Opts}; +use crate::config::Config; +use crate::exec::CommandSet; +use crate::exit_codes::ExitCode; +use crate::filetypes::FileTypes; +#[cfg(unix)] +use crate::filter::OwnerFilter; +use crate::filter::TimeFilter; +use crate::patterns::{ + build_patterns, pattern_has_uppercase_char, patterns_match_strings_with_leading_dots, +}; + +// We use jemalloc for performance reasons, see https://github.com/sharkdp/fd/pull/481 +// FIXME: re-enable jemalloc on macOS, see comment in Cargo.toml file for more infos +// This has to be kept in sync with the Cargo.toml file section that declares a +// dependency on tikv-jemallocator. +#[cfg(all( + not(windows), + not(target_os = "android"), + not(target_os = "macos"), + not(target_os = "freebsd"), + not(target_os = "openbsd"), + not(target_os = "illumos"), + not(all(target_env = "musl", target_pointer_width = "32")), + not(target_arch = "riscv64"), + feature = "use-jemalloc" +))] +#[global_allocator] +static ALLOC: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc; + +// vivid --color-mode 8-bit generate molokai +const DEFAULT_LS_COLORS: &str = " +ow=0:or=0;38;5;16;48;5;203:no=0:ex=1;38;5;203:cd=0;38;5;203;48;5;236:mi=0;38;5;16;48;5;203:*~=0;38;5;243:st=0:pi=0;38;5;16;48;5;81:fi=0:di=0;38;5;81:so=0;38;5;16;48;5;203:bd=0;38;5;81;48;5;236:tw=0:ln=0;38;5;203:*.m=0;38;5;48:*.o=0;38;5;243:*.z=4;38;5;203:*.a=1;38;5;203:*.r=0;38;5;48:*.c=0;38;5;48:*.d=0;38;5;48:*.t=0;38;5;48:*.h=0;38;5;48:*.p=0;38;5;48:*.cc=0;38;5;48:*.ll=0;38;5;48:*.jl=0;38;5;48:*css=0;38;5;48:*.md=0;38;5;185:*.gz=4;38;5;203:*.nb=0;38;5;48:*.mn=0;38;5;48:*.go=0;38;5;48:*.xz=4;38;5;203:*.so=1;38;5;203:*.rb=0;38;5;48:*.pm=0;38;5;48:*.bc=0;38;5;243:*.py=0;38;5;48:*.as=0;38;5;48:*.pl=0;38;5;48:*.rs=0;38;5;48:*.sh=0;38;5;48:*.7z=4;38;5;203:*.ps=0;38;5;186:*.cs=0;38;5;48:*.el=0;38;5;48:*.rm=0;38;5;208:*.hs=0;38;5;48:*.td=0;38;5;48:*.ui=0;38;5;149:*.ex=0;38;5;48:*.js=0;38;5;48:*.cp=0;38;5;48:*.cr=0;38;5;48:*.la=0;38;5;243:*.kt=0;38;5;48:*.ml=0;38;5;48:*.vb=0;38;5;48:*.gv=0;38;5;48:*.lo=0;38;5;243:*.hi=0;38;5;243:*.ts=0;38;5;48:*.ko=1;38;5;203:*.hh=0;38;5;48:*.pp=0;38;5;48:*.di=0;38;5;48:*.bz=4;38;5;203:*.fs=0;38;5;48:*.png=0;38;5;208:*.zsh=0;38;5;48:*.mpg=0;38;5;208:*.pid=0;38;5;243:*.xmp=0;38;5;149:*.iso=4;38;5;203:*.m4v=0;38;5;208:*.dot=0;38;5;48:*.ods=0;38;5;186:*.inc=0;38;5;48:*.sxw=0;38;5;186:*.aif=0;38;5;208:*.git=0;38;5;243:*.gvy=0;38;5;48:*.tbz=4;38;5;203:*.log=0;38;5;243:*.txt=0;38;5;185:*.ico=0;38;5;208:*.csx=0;38;5;48:*.vob=0;38;5;208:*.pgm=0;38;5;208:*.pps=0;38;5;186:*.ics=0;38;5;186:*.img=4;38;5;203:*.fon=0;38;5;208:*.hpp=0;38;5;48:*.bsh=0;38;5;48:*.sql=0;38;5;48:*TODO=1:*.php=0;38;5;48:*.pkg=4;38;5;203:*.ps1=0;38;5;48:*.csv=0;38;5;185:*.ilg=0;38;5;243:*.ini=0;38;5;149:*.pyc=0;38;5;243:*.psd=0;38;5;208:*.htc=0;38;5;48:*.swp=0;38;5;243:*.mli=0;38;5;48:*hgrc=0;38;5;149:*.bst=0;38;5;149:*.ipp=0;38;5;48:*.fsi=0;38;5;48:*.tcl=0;38;5;48:*.exs=0;38;5;48:*.out=0;38;5;243:*.jar=4;38;5;203:*.xls=0;38;5;186:*.ppm=0;38;5;208:*.apk=4;38;5;203:*.aux=0;38;5;243:*.rpm=4;38;5;203:*.dll=1;38;5;203:*.eps=0;38;5;208:*.exe=1;38;5;203:*.doc=0;38;5;186:*.wma=0;38;5;208:*.deb=4;38;5;203:*.pod=0;38;5;48:*.ind=0;38;5;243:*.nix=0;38;5;149:*.lua=0;38;5;48:*.epp=0;38;5;48:*.dpr=0;38;5;48:*.htm=0;38;5;185:*.ogg=0;38;5;208:*.bin=4;38;5;203:*.otf=0;38;5;208:*.yml=0;38;5;149:*.pro=0;38;5;149:*.cxx=0;38;5;48:*.tex=0;38;5;48:*.fnt=0;38;5;208:*.erl=0;38;5;48:*.sty=0;38;5;243:*.bag=4;38;5;203:*.rst=0;38;5;185:*.pdf=0;38;5;186:*.pbm=0;38;5;208:*.xcf=0;38;5;208:*.clj=0;38;5;48:*.gif=0;38;5;208:*.rar=4;38;5;203:*.elm=0;38;5;48:*.bib=0;38;5;149:*.tsx=0;38;5;48:*.dmg=4;38;5;203:*.tmp=0;38;5;243:*.bcf=0;38;5;243:*.mkv=0;38;5;208:*.svg=0;38;5;208:*.cpp=0;38;5;48:*.vim=0;38;5;48:*.bmp=0;38;5;208:*.ltx=0;38;5;48:*.fls=0;38;5;243:*.flv=0;38;5;208:*.wav=0;38;5;208:*.m4a=0;38;5;208:*.mid=0;38;5;208:*.hxx=0;38;5;48:*.pas=0;38;5;48:*.wmv=0;38;5;208:*.tif=0;38;5;208:*.kex=0;38;5;186:*.mp4=0;38;5;208:*.bak=0;38;5;243:*.xlr=0;38;5;186:*.dox=0;38;5;149:*.swf=0;38;5;208:*.tar=4;38;5;203:*.tgz=4;38;5;203:*.cfg=0;38;5;149:*.xml=0; +38;5;185:*.jpg=0;38;5;208:*.mir=0;38;5;48:*.sxi=0;38;5;186:*.bz2=4;38;5;203:*.odt=0;38;5;186:*.mov=0;38;5;208:*.toc=0;38;5;243:*.bat=1;38;5;203:*.asa=0;38;5;48:*.awk=0;38;5;48:*.sbt=0;38;5;48:*.vcd=4;38;5;203:*.kts=0;38;5;48:*.arj=4;38;5;203:*.blg=0;38;5;243:*.c++=0;38;5;48:*.odp=0;38;5;186:*.bbl=0;38;5;243:*.idx=0;38;5;243:*.com=1;38;5;203:*.mp3=0;38;5;208:*.avi=0;38;5;208:*.def=0;38;5;48:*.cgi=0;38;5;48:*.zip=4;38;5;203:*.ttf=0;38;5;208:*.ppt=0;38;5;186:*.tml=0;38;5;149:*.fsx=0;38;5;48:*.h++=0;38;5;48:*.rtf=0;38;5;186:*.inl=0;38;5;48:*.yaml=0;38;5;149:*.html=0;38;5;185:*.mpeg=0;38;5;208:*.java=0;38;5;48:*.hgrc=0;38;5;149:*.orig=0;38;5;243:*.conf=0;38;5;149:*.dart=0;38;5;48:*.psm1=0;38;5;48:*.rlib=0;38;5;243:*.fish=0;38;5;48:*.bash=0;38;5;48:*.make=0;38;5;149:*.docx=0;38;5;186:*.json=0;38;5;149:*.psd1=0;38;5;48:*.lisp=0;38;5;48:*.tbz2=4;38;5;203:*.diff=0;38;5;48:*.epub=0;38;5;186:*.xlsx=0;38;5;186:*.pptx=0;38;5;186:*.toml=0;38;5;149:*.h264=0;38;5;208:*.purs=0;38;5;48:*.flac=0;38;5;208:*.tiff=0;38;5;208:*.jpeg=0;38;5;208:*.lock=0;38;5;243:*.less=0;38;5;48:*.dyn_o=0;38;5;243:*.scala=0;38;5;48:*.mdown=0;38;5;185:*.shtml=0;38;5;185:*.class=0;38;5;243:*.cache=0;38;5;243:*.cmake=0;38;5;149:*passwd=0;38;5;149:*.swift=0;38;5;48:*shadow=0;38;5;149:*.xhtml=0;38;5;185:*.patch=0;38;5;48:*.cabal=0;38;5;48:*README=0;38;5;16;48;5;186:*.toast=4;38;5;203:*.ipynb=0;38;5;48:*COPYING=0;38;5;249:*.gradle=0;38;5;48:*.matlab=0;38;5;48:*.config=0;38;5;149:*LICENSE=0;38;5;249:*.dyn_hi=0;38;5;243:*.flake8=0;38;5;149:*.groovy=0;38;5;48:*INSTALL=0;38;5;16;48;5;186:*TODO.md=1:*.ignore=0;38;5;149:*Doxyfile=0;38;5;149:*TODO.txt=1:*setup.py=0;38;5;149:*Makefile=0;38;5;149:*.gemspec=0;38;5;149:*.desktop=0;38;5;149:*.rgignore=0;38;5;149:*.markdown=0;38;5;185:*COPYRIGHT=0;38;5;249:*configure=0;38;5;149:*.DS_Store=0;38;5;243:*.kdevelop=0;38;5;149:*.fdignore=0;38;5;149:*README.md=0;38;5;16;48;5;186:*.cmake.in=0;38;5;149:*SConscript=0;38;5;149:*CODEOWNERS=0;38;5;149:*.localized=0;38;5;243:*.gitignore=0;38;5;149:*Dockerfile=0;38;5;149:*.gitconfig=0;38;5;149:*INSTALL.md=0;38;5;16;48;5;186:*README.txt=0;38;5;16;48;5;186:*SConstruct=0;38;5;149:*.scons_opt=0;38;5;243:*.travis.yml=0;38;5;186:*.gitmodules=0;38;5;149:*.synctex.gz=0;38;5;243:*LICENSE-MIT=0;38;5;249:*MANIFEST.in=0;38;5;149:*Makefile.in=0;38;5;243:*Makefile.am=0;38;5;149:*INSTALL.txt=0;38;5;16;48;5;186:*configure.ac=0;38;5;149:*.applescript=0;38;5;48:*appveyor.yml=0;38;5;186:*.fdb_latexmk=0;38;5;243:*CONTRIBUTORS=0;38;5;16;48;5;186:*.clang-format=0;38;5;149:*LICENSE-APACHE=0;38;5;249:*CMakeLists.txt=0;38;5;149:*CMakeCache.txt=0;38;5;243:*.gitattributes=0;38;5;149:*CONTRIBUTORS.md=0;38;5;16;48;5;186:*.sconsign.dblite=0;38;5;243:*requirements.txt=0;38;5;149:*CONTRIBUTORS.txt=0;38;5;16;48;5;186:*package-lock.json=0;38;5;243:*.CFUserTextEncoding=0;38;5;243 +"; + +fn main() { + let result = run(); + match result { + Ok(exit_code) => { + exit_code.exit(); + } + Err(err) => { + // NB: we use eprintln directly instead of print_error!() because + // we sanitize anyhow errors at the generation site + eprintln!("[fd error]: {:#}", err); + ExitCode::GeneralError.exit(); + } + } +} + +fn run() -> Result { + let mut opts = Opts::parse(); + + #[cfg(feature = "completions")] + if let Some(shell) = opts.gen_completions()? { + return print_completions(shell); + } + + set_working_dir(&opts)?; + let search_paths = opts.search_paths()?; + if search_paths.is_empty() { + bail!("No valid search paths given."); + } + + ensure_search_pattern_is_not_a_path(&opts)?; + let mut patterns = opts.exprs.take().unwrap_or(Vec::new()); + if let Some(pattern) = opts.pattern.take() { + patterns.push(pattern); + } + + let pattern_type = determine_pattern_type(&opts); + // The search will be case-sensitive if the command line flag is set or + // if any of the patterns has an uppercase character (smart case). + let ignore_case = opts.ignore_case + || !(opts.case_sensitive + || patterns + .iter() + .any(|pat| pattern_has_uppercase_char(pattern_type, pat))); + + let config = construct_config(opts)?; + + ensure_use_hidden_option_for_leading_dot_pattern(&patterns, &config, pattern_type)?; + + let matcher = build_patterns(patterns, pattern_type, ignore_case)?; + + walk::scan(&search_paths, matcher, config) +} + +#[cfg(feature = "completions")] +#[cold] +fn print_completions(shell: clap_complete::Shell) -> Result { + // The program name is the first argument. + let first_arg = env::args().next(); + let program_name = first_arg + .as_ref() + .map(Path::new) + .and_then(|path| path.file_stem()) + .and_then(|file| file.to_str()) + .unwrap_or("fd"); + let mut cmd = Opts::command(); + cmd.build(); + clap_complete::generate(shell, &mut cmd, program_name, &mut std::io::stdout()); + Ok(ExitCode::Success) +} + +fn set_working_dir(opts: &Opts) -> Result<()> { + if let Some(ref base_directory) = opts.base_directory { + if !filesystem::is_existing_directory(base_directory) { + return Err(anyhow!( + "The '--base-directory' path {:?} is not a directory.", + base_directory.to_string_lossy().as_ref() + )); + } + env::set_current_dir(base_directory).with_context(|| { + format!( + "Could not set {:?} as the current working directory", + base_directory.to_string_lossy().as_ref() + ) + })?; + } + Ok(()) +} + +/// Detect if the user accidentally supplied a path instead of a search pattern. +/// +/// Without `--full-path`, fd matches patterns against file names, so any pattern +/// containing a path separator can never match. This applies to the primary +/// positional pattern *and* to every `--and` pattern, since all of them are +/// chained together and matched against the file name (see `run`). Two cases per +/// pattern are worth a friendly error rather than silent "no results": +/// +/// 1. The pattern contains '/'. '/' is always a path separator (including on +/// Windows) and has no regex meaning, so flagging it is safe and catches the +/// common Linux/macOS mistake of pasting a full path as the pattern. +/// 2. On Windows only, the pattern contains the native `\` separator *and* +/// names an existing directory on disk. We can't treat `\` as a pure +/// path-separator signal there because it is also the regex escape char, +/// so valid regex patterns like `\Ac` or `\d+` must still run. Requiring +/// that the pattern resolves to a real directory avoids those false +/// positives while preserving the legacy diagnostic for operators who +/// literally typed a directory path. +/// +/// See https://github.com/sharkdp/fd/issues/1873. +fn ensure_search_pattern_is_not_a_path(opts: &Opts) -> Result<()> { + if opts.full_path { + return Ok(()); + } + if let Some(ref pattern) = opts.pattern { + // Check the primary pattern and every `--and` pattern. They are all matched + // against the file name (see `run`), so a path separator in any of them is + // the same silent "no results" footgun. + for pattern in std::iter::once(pattern).chain(opts.exprs.iter().flatten()) { + ensure_single_search_pattern_is_not_a_path(pattern)?; + } + } + Ok(()) +} + +/// Apply the path-separator diagnostic to a single pattern. See +/// [`ensure_search_pattern_is_not_a_path`] for the rationale of each case. +fn ensure_single_search_pattern_is_not_a_path(pattern: &str) -> Result<()> { + // Start with the cheap check: '/' is always a path separator, including on + // Windows, and has no regex meaning, so flagging it is safe and catches the + // Linux/macOS mistake of pasting a full path as the pattern. + #[cfg_attr(not(windows), allow(unused_mut))] + let mut should_warn = pattern.contains('/'); + + // On Windows we additionally accept the native `\` separator, but only when + // the pattern actually resolves to an existing directory - `\` is also the + // regex escape char there, so valid patterns like `\Ac` or `\d+` must still + // run. The is_dir syscall is only needed when `should_warn` is still false, + // so short-circuit via `||` to avoid the stat call on the happy path. + #[cfg(windows)] + { + should_warn = should_warn + || (pattern.contains(std::path::MAIN_SEPARATOR) && Path::new(pattern).is_dir()); + } + + if should_warn { + Err(anyhow!( + "The search pattern '{pattern}' contains a path-separation character \ + and will not lead to any search results.\n\n\ + If you want to search for all files inside the '{pattern}' directory, use a match-all pattern:\n\n \ + fd . '{pattern}'\n\n\ + Instead, if you want your pattern to match the full file path, use:\n\n \ + fd --full-path '{pattern}'", + pattern = pattern, + )) + } else { + Ok(()) + } +} + +fn determine_pattern_type(opts: &Opts) -> PatternType { + if opts.glob { + PatternType::Glob + } else if opts.exact { + PatternType::Exact + } else if opts.fixed_strings { + PatternType::Fixed + } else { + PatternType::Regex + } +} + +fn check_path_separator_length(path_separator: Option<&str>) -> Result<()> { + match (cfg!(windows), path_separator) { + (true, Some(sep)) if sep.len() > 1 => Err(anyhow!( + "A path separator must be exactly one byte, but \ + the given separator is {} bytes: '{}'.\n\ + In some shells on Windows, '/' is automatically \ + expanded. Try to use '//' instead.", + sep.len(), + sep + )), + _ => Ok(()), + } +} + +fn construct_config(mut opts: Opts) -> Result { + let path_separator = opts + .path_separator + .take() + .or_else(filesystem::default_path_separator); + let actual_path_separator = path_separator + .clone() + .unwrap_or_else(|| std::path::MAIN_SEPARATOR.to_string()); + check_path_separator_length(path_separator.as_deref())?; + + let size_limits = std::mem::take(&mut opts.size); + let time_constraints = extract_time_constraints(&opts)?; + #[cfg(unix)] + let owner_constraint: Option = opts.owner.and_then(OwnerFilter::filter_ignore); + + #[cfg(windows)] + let ansi_colors_support = + nu_ansi_term::enable_ansi_support().is_ok() || std::env::var_os("TERM").is_some(); + #[cfg(not(windows))] + let ansi_colors_support = true; + + let interactive_terminal = std::io::stdout().is_terminal(); + + let colored_output = match opts.color { + ColorWhen::Always => true, + ColorWhen::Never => false, + ColorWhen::Auto => { + let no_color = env::var_os("NO_COLOR").is_some_and(|x| !x.is_empty()); + ansi_colors_support && !no_color && interactive_terminal + } + }; + + let ls_colors = if colored_output { + Some(LsColors::from_env().unwrap_or_else(|| LsColors::from_string(DEFAULT_LS_COLORS))) + } else { + None + }; + let hyperlink = match opts.hyperlink { + HyperlinkWhen::Always => true, + HyperlinkWhen::Never => false, + HyperlinkWhen::Auto => colored_output, + }; + let command = extract_command(&mut opts, colored_output)?; + let has_command = command.is_some(); + + let full_path_base = if opts.full_path { + Some(env::current_dir().context( + "Could not determine current directory. \ + This is required for --full-path.", + )?) + } else { + None + }; + + Ok(Config { + full_path_base, + ignore_hidden: !opts.hidden, + read_fdignore: !opts.no_ignore, + read_vcsignore: !(opts.no_ignore || opts.no_ignore_vcs), + require_git_to_read_vcsignore: !opts.no_require_git, + read_parent_ignore: !opts.no_ignore_parent, + read_global_ignore: !(opts.no_ignore || opts.no_global_ignore_file), + follow_links: opts.follow, + #[cfg(any(unix, windows))] + one_file_system: opts.one_file_system, + #[cfg(target_os = "wasi")] + one_file_system: false, + null_separator: opts.null_separator, + quiet: opts.quiet, + max_depth: opts.max_depth(), + min_depth: opts.min_depth(), + prune: opts.prune, + threads: if cfg!(target_os = "wasi") { 1 } else { opts.threads().get() }, + max_buffer_time: opts.max_buffer_time, + ls_colors, + hyperlink, + interactive_terminal, + file_types: opts + .filetype + .as_ref() + .map(|values| FileTypes::from_types(values)), + extensions: opts + .extensions + .as_ref() + .map(|exts| { + let patterns = exts + .iter() + .map(|e| e.trim_start_matches('.')) + .map(|e| format!(r".\.{}$", regex::escape(e))); + RegexSetBuilder::new(patterns) + .case_insensitive(true) + .build() + }) + .transpose()?, + format: opts + .format + .as_deref() + .map(crate::fmt::FormatTemplate::parse), + command: command.map(Arc::new), + batch_size: opts.batch_size, + exclude_patterns: opts.exclude.iter().map(|p| String::from("!") + p).collect(), + ignore_files: std::mem::take(&mut opts.ignore_file), + size_constraints: size_limits, + time_constraints, + #[cfg(unix)] + owner_constraint, + show_filesystem_errors: opts.show_errors, + path_separator, + actual_path_separator, + max_results: opts.max_results(), + strip_cwd_prefix: opts.strip_cwd_prefix(|| !(opts.null_separator || has_command)), + ignore_contain: opts.ignore_contain, + }) +} + +fn extract_command(opts: &mut Opts, colored_output: bool) -> Result> { + opts.exec + .command + .take() + .map(Ok) + .or_else(|| { + if !opts.list_details { + return None; + } + + let res = determine_ls_command(colored_output) + .map(|cmd| CommandSet::new_batch([cmd]).unwrap()); + Some(res) + }) + .transpose() +} + +fn determine_ls_command(colored_output: bool) -> Result> { + #[allow(unused)] + let gnu_ls = |command_name| { + let color_arg = if colored_output { + "--color=always" + } else { + "--color=never" + }; + // Note: we use short options here (instead of --long-options) to support more + // platforms (like BusyBox). + vec![ + command_name, + "-l", // long listing format + "-h", // human readable file sizes + "-d", // list directories themselves, not their contents + color_arg, + ] + }; + let cmd: Vec<&str> = if cfg!(unix) { + if !cfg!(any( + target_os = "macos", + target_os = "dragonfly", + target_os = "freebsd", + target_os = "netbsd", + target_os = "openbsd" + )) { + // Assume ls is GNU ls + gnu_ls("ls") + } else { + // macOS, DragonFlyBSD, FreeBSD + use std::process::{Command, Stdio}; + + // Use GNU ls, if available (support for --color=auto, better LS_COLORS support) + let gnu_ls_exists = Command::new("gls") + .arg("--version") + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .status() + .is_ok(); + + if gnu_ls_exists { + gnu_ls("gls") + } else { + let mut cmd = vec![ + "ls", // BSD version of ls + "-l", // long listing format + "-h", // '--human-readable' is not available, '-h' is + "-d", // '--directory' is not available, but '-d' is + ]; + + if !cfg!(any(target_os = "netbsd", target_os = "openbsd")) && colored_output { + // -G is not available in NetBSD's and OpenBSD's ls + cmd.push("-G"); + } + + cmd + } + } + } else if cfg!(windows) { + use std::process::{Command, Stdio}; + + // Use GNU ls, if available + let gnu_ls_exists = Command::new("ls") + .arg("--version") + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .status() + .is_ok(); + + if gnu_ls_exists { + gnu_ls("ls") + } else { + return Err(anyhow!( + "'fd --list-details' is not supported on Windows unless GNU 'ls' is installed." + )); + } + } else { + return Err(anyhow!( + "'fd --list-details' is not supported on this platform." + )); + }; + Ok(cmd) +} + +fn extract_time_constraints(opts: &Opts) -> Result> { + let mut time_constraints: Vec = Vec::new(); + if let Some(ref t) = opts.changed_within { + if let Some(f) = TimeFilter::after(t) { + time_constraints.push(f); + } else { + return Err(anyhow!( + "'{}' is not a valid date or duration. See 'fd --help'.", + t + )); + } + } + if let Some(ref t) = opts.changed_before { + if let Some(f) = TimeFilter::before(t) { + time_constraints.push(f); + } else { + return Err(anyhow!( + "'{}' is not a valid date or duration. See 'fd --help'.", + t + )); + } + } + Ok(time_constraints) +} + +fn ensure_use_hidden_option_for_leading_dot_pattern( + patterns: &[String], + config: &Config, + pattern_type: PatternType, +) -> Result<()> { + if cfg!(unix) + && config.ignore_hidden + && patterns_match_strings_with_leading_dots(patterns, pattern_type) + { + Err(anyhow!( + "The pattern(s) seems to only match files with a leading dot, but hidden files are \ + filtered by default. Consider adding -H/--hidden to search hidden files as well \ + or adjust your search pattern(s)." + )) + } else { + Ok(()) + } +} diff --git a/userland/fd/src/output.rs b/userland/fd/src/output.rs new file mode 100644 index 0000000..ce1e793 --- /dev/null +++ b/userland/fd/src/output.rs @@ -0,0 +1,196 @@ +use std::borrow::Cow; +use std::io::{self, Write}; + +use lscolors::{Indicator, LsColors, Style}; + +use crate::config::Config; +use crate::dir_entry::DirEntry; +use crate::fmt::FormatTemplate; +use crate::hyperlink::PathUrl; +use crate::sanitize::sanitize_for_term; + +fn replace_path_separator(path: &str, new_path_separator: &str) -> String { + path.replace(std::path::MAIN_SEPARATOR, new_path_separator) +} + +// TODO: this function is performance critical and can probably be optimized +pub fn print_entry(stdout: &mut W, entry: &DirEntry, config: &Config) -> io::Result<()> { + let mut has_hyperlink = false; + if config.hyperlink + && let Some(url) = PathUrl::new(entry.path()) + { + write!(stdout, "\x1B]8;;{url}\x1B\\")?; + has_hyperlink = true; + } + + if let Some(ref format) = config.format { + print_entry_format(stdout, entry, config, format)?; + } else if let Some(ref ls_colors) = config.ls_colors { + print_entry_colorized(stdout, entry, config, ls_colors)?; + } else { + print_entry_uncolorized(stdout, entry, config)?; + }; + + if has_hyperlink { + write!(stdout, "\x1B]8;;\x1B\\")?; + } + + if config.null_separator { + write!(stdout, "\0") + } else { + writeln!(stdout) + } +} + +// Display a trailing slash if the path is a directory and the config option is enabled. +// If the path_separator option is set, display that instead. +// The trailing slash will not be colored. +#[inline] +fn print_trailing_slash( + stdout: &mut W, + entry: &DirEntry, + config: &Config, + style: Option<&Style>, +) -> io::Result<()> { + if entry.file_type().is_some_and(|ft| ft.is_dir()) { + write!( + stdout, + "{}", + style + .map(Style::to_nu_ansi_term_style) + .unwrap_or_default() + .paint(&config.actual_path_separator) + )?; + } + Ok(()) +} + +// TODO: this function is performance critical and can probably be optimized +fn print_entry_format( + stdout: &mut W, + entry: &DirEntry, + config: &Config, + format: &FormatTemplate, +) -> io::Result<()> { + let output = format.generate( + entry.stripped_path(config), + config.path_separator.as_deref(), + ); + // TODO: support writing raw bytes on unix? + let s = output.to_string_lossy(); + write!( + stdout, + "{}", + sanitize_for_term(&s, config.interactive_terminal) + ) +} + +// TODO: this function is performance critical and can probably be optimized +fn print_entry_colorized( + stdout: &mut W, + entry: &DirEntry, + config: &Config, + ls_colors: &LsColors, +) -> io::Result<()> { + let mut offset = 0; + let path = entry.stripped_path(config); + let path_str = path.to_string_lossy(); + + if let Some(parent) = path.parent() { + offset = parent.to_string_lossy().len(); + for c in path_str[offset..].chars() { + if std::path::is_separator(c) { + offset += c.len_utf8(); + } else { + break; + } + } + } + + if offset > 0 { + let mut parent_str = Cow::from(&path_str[..offset]); + if let Some(ref separator) = config.path_separator { + *parent_str.to_mut() = replace_path_separator(&parent_str, separator); + } + + let style = ls_colors + .style_for_indicator(Indicator::Directory) + .map(Style::to_nu_ansi_term_style) + .unwrap_or_default(); + let safe_parent = sanitize_for_term(&parent_str, config.interactive_terminal); + // We explicitly use prefix/suffix instead of paint so that we can use Sanitized + write!( + stdout, + "{}{}{}", + style.prefix(), + safe_parent, + style.suffix() + )?; + } + + let style = entry + .style(ls_colors) + .map(Style::to_nu_ansi_term_style) + .unwrap_or_default(); + let safe_basename = sanitize_for_term(&path_str[offset..], config.interactive_terminal); + // We explicitly use prefix/suffix instead of paint so that we can use Sanitized + write!( + stdout, + "{}{}{}", + style.prefix(), + safe_basename, + style.suffix() + )?; + + print_trailing_slash( + stdout, + entry, + config, + ls_colors.style_for_indicator(Indicator::Directory), + )?; + + Ok(()) +} + +// TODO: this function is performance critical and can probably be optimized +fn print_entry_uncolorized_base( + stdout: &mut W, + entry: &DirEntry, + config: &Config, +) -> io::Result<()> { + let path = entry.stripped_path(config); + + let mut path_string = path.to_string_lossy(); + if let Some(ref separator) = config.path_separator { + *path_string.to_mut() = replace_path_separator(&path_string, separator); + } + let safe = sanitize_for_term(&path_string, config.interactive_terminal); + write!(stdout, "{safe}")?; + print_trailing_slash(stdout, entry, config, None) +} + +#[cfg(not(unix))] +fn print_entry_uncolorized( + stdout: &mut W, + entry: &DirEntry, + config: &Config, +) -> io::Result<()> { + print_entry_uncolorized_base(stdout, entry, config) +} + +#[cfg(unix)] +fn print_entry_uncolorized( + stdout: &mut W, + entry: &DirEntry, + config: &Config, +) -> io::Result<()> { + use std::os::unix::ffi::OsStrExt; + + if config.interactive_terminal || config.path_separator.is_some() { + print_entry_uncolorized_base(stdout, entry, config) + } else { + // Piped output: raw bytes so invalid UTF-8 filenames reach downstream tools intact. + stdout.write_all(entry.stripped_path(config).as_os_str().as_bytes())?; + print_trailing_slash(stdout, entry, config, None) + } +} diff --git a/userland/fd/src/patterns.rs b/userland/fd/src/patterns.rs new file mode 100644 index 0000000..5211946 --- /dev/null +++ b/userland/fd/src/patterns.rs @@ -0,0 +1,177 @@ +use std::ffi::OsStr; + +use anyhow::{Result, anyhow}; +use globset::{Glob, GlobBuilder, GlobMatcher, GlobSet, GlobSetBuilder}; +use memchr::memmem; +use regex::bytes::{RegexSet, RegexSetBuilder}; + +use crate::regex_helper::{self, pattern_matches_strings_with_leading_dot}; + +pub trait Matcher { + fn matches_path(&self, path: &OsStr) -> bool; +} + +pub type Patterns = Box; + +#[derive(Eq, PartialEq, Copy, Clone)] +pub enum PatternType { + Regex, + Fixed, + Glob, + Exact, +} + +impl Matcher for RegexSet { + fn matches_path(&self, path: &OsStr) -> bool { + let haystack = path.as_encoded_bytes(); + let matches = self.matches(haystack); + // Return true if the number of regexes that matched + // equals the total number of regexes. + matches.iter().count() == self.len() + } +} + +impl Matcher for GlobSet { + fn matches_path(&self, path: &OsStr) -> bool { + self.matches_all(path) + } +} + +/// In the common case a single glob, it is simpler, and +/// faster to just use a single Glob instead of a GlobSet +impl Matcher for GlobMatcher { + fn matches_path(&self, path: &OsStr) -> bool { + self.is_match(path) + } +} + +/// Matcher that matches fixed strings +pub struct FixedStrings(pub Vec); + +impl Matcher for FixedStrings { + fn matches_path(&self, path: &OsStr) -> bool { + let path = path.as_encoded_bytes(); + self.0.iter().all(|f| bytes_contains(path, f.as_bytes())) + } +} + +pub struct ExactStrings(pub Vec); + +impl Matcher for ExactStrings { + fn matches_path(&self, path: &OsStr) -> bool { + self.0.iter().all(|s| s.as_str() == path) + } +} + +/// Matcher that matches everything +pub struct MatchAll; +impl Matcher for MatchAll { + fn matches_path(&self, _path: &OsStr) -> bool { + true + } +} +pub fn build_patterns( + mut patterns: Vec, + pattern_type: PatternType, + ignore_case: bool, +) -> Result { + if patterns.is_empty() || is_empty_pattern(&patterns, pattern_type) { + return Ok(Box::new(MatchAll)); + } + match pattern_type { + PatternType::Glob => build_glob_matcher(patterns, ignore_case), + PatternType::Fixed if !ignore_case => Ok(Box::new(FixedStrings(patterns))), + PatternType::Exact if !ignore_case => Ok(Box::new(ExactStrings(patterns))), + typ => { + // TODO: is there a better way we could handle case insensitive fixed strings? + if typ == PatternType::Fixed { + for pattern in patterns.iter_mut() { + *pattern = regex::escape(pattern); + } + } else if typ == PatternType::Exact { + // Would using eq_ignore_ascii case be good enough? + for pattern in patterns.iter_mut() { + // Anchor the escaped pattern so the full filename (or path) must match exactly. + // Literal. No substring matching. + *pattern = format!("^{}$", regex::escape(pattern)); + } + } + Ok(Box::new(build_regex_matcher(patterns, ignore_case)?)) + } + } +} + +pub fn patterns_match_strings_with_leading_dots( + patterns: &[String], + pattern_type: PatternType, +) -> bool { + let mut iter = patterns.iter(); + match pattern_type { + PatternType::Regex => iter.any(|pat| pattern_matches_strings_with_leading_dot(pat)), + // fixed strings aren't anchored so always false + PatternType::Fixed => false, + // globs and exact just check if it starts with a . + PatternType::Glob | PatternType::Exact => patterns.iter().any(|pat| pat.starts_with(".")), + } +} + +pub fn pattern_has_uppercase_char(pattern_type: PatternType, pattern: &str) -> bool { + match pattern_type { + PatternType::Regex => regex_helper::pattern_has_uppercase_char(pattern), + // For everything else it's sufficient to just check if there are any uppercase letters + _ => pattern.chars().any(char::is_uppercase), + } +} + +fn build_glob_matcher(patterns: Vec, ignore_case: bool) -> Result { + Ok(if patterns.len() == 1 { + Box::new(build_glob(&patterns[0], ignore_case)?.compile_matcher()) + } else { + let mut builder = GlobSetBuilder::new(); + for pat in patterns { + builder.add(build_glob(&pat, ignore_case)?); + } + Box::new(builder.build()?) + }) +} + +fn build_glob(pattern: &str, ignore_case: bool) -> Result { + Ok(GlobBuilder::new(pattern) + .literal_separator(true) + .case_insensitive(ignore_case) + .build()?) +} + +fn build_regex_matcher(patterns: Vec, ignore_case: bool) -> Result { + RegexSetBuilder::new(patterns) + .case_insensitive(ignore_case) + .dot_matches_new_line(true) + .build() + .map_err(|e| { + anyhow!( + "{}\n\nNote: You can search for literal substrings with '--fixed-strings' \ + or literal strings with '--exact' options (instead of a regular expression). \ + Alternatively, you can also use the '--glob' option to match on a glob pattern.", + e, + ) + }) +} + +fn is_empty_pattern(patterns: &[String], pattern_type: PatternType) -> bool { + pattern_type != PatternType::Exact && patterns.len() == 1 && patterns[0].is_empty() +} + +/// Test if the needle is a substring of the haystack +fn bytes_contains(haystack: &[u8], needle: &[u8]) -> bool { + memmem::find(haystack, needle).is_some() +} + +#[test] +fn empty_matches_all() { + let patterns_none = build_patterns(Vec::new(), PatternType::Glob, false).unwrap(); + let patterns_empty_str = + build_patterns(vec!["".to_string()], PatternType::Glob, false).unwrap(); + + assert!(patterns_none.matches_path("abcd".as_ref())); + assert!(patterns_empty_str.matches_path("abcd".as_ref())); +} diff --git a/userland/fd/src/regex_helper.rs b/userland/fd/src/regex_helper.rs new file mode 100644 index 0000000..1b4a5de --- /dev/null +++ b/userland/fd/src/regex_helper.rs @@ -0,0 +1,130 @@ +use regex_syntax::ParserBuilder; +use regex_syntax::hir::Hir; + +/// Determine if a regex pattern contains a literal uppercase character. +pub fn pattern_has_uppercase_char(pattern: &str) -> bool { + // The HIR includes case-folded alternatives introduced by inline flags. + // Smart case must consider the original literals, before case folding. + regex_syntax::ast::parse::Parser::new() + .parse(pattern) + .map(|ast| regex_syntax::ast::visit(&ast, UppercaseVisitor(false)).unwrap_or(false)) + .unwrap_or(false) +} + +struct UppercaseVisitor(bool); + +impl regex_syntax::ast::Visitor for UppercaseVisitor { + type Output = bool; + type Err = std::convert::Infallible; + + fn finish(self) -> Result { + Ok(self.0) + } + + fn visit_pre(&mut self, ast: ®ex_syntax::ast::Ast) -> Result<(), Self::Err> { + if let regex_syntax::ast::Ast::Literal(literal) = ast { + self.0 |= literal.c.is_uppercase(); + } + Ok(()) + } + + fn visit_class_set_item_pre( + &mut self, + item: ®ex_syntax::ast::ClassSetItem, + ) -> Result<(), Self::Err> { + use regex_syntax::ast::ClassSetItem; + + self.0 |= match item { + ClassSetItem::Literal(literal) => literal.c.is_uppercase(), + ClassSetItem::Range(range) => { + range.start.c.is_uppercase() || range.end.c.is_uppercase() + } + _ => false, + }; + Ok(()) + } +} + +/// Determine if a regex pattern only matches strings starting with a literal dot (hidden files) +pub fn pattern_matches_strings_with_leading_dot(pattern: &str) -> bool { + let mut parser = ParserBuilder::new().utf8(false).build(); + + parser + .parse(pattern) + .map(|hir| hir_matches_strings_with_leading_dot(&hir)) + .unwrap_or(false) +} + +/// See above. +fn hir_matches_strings_with_leading_dot(hir: &Hir) -> bool { + use regex_syntax::hir::*; + + // Note: this only really detects the simplest case where a regex starts with + // "^\\.", i.e. a start text anchor and a literal dot character. There are a lot + // of other patterns that ONLY match hidden files, e.g. ^(\\.foo|\\.bar) which are + // not (yet) detected by this algorithm. + match hir.kind() { + HirKind::Concat(hirs) => { + let mut hirs = hirs.iter(); + if let Some(hir) = hirs.next() { + if hir.kind() != &HirKind::Look(Look::Start) { + return false; + } + } else { + return false; + } + + if let Some(hir) = hirs.next() { + match hir.kind() { + HirKind::Literal(Literal(bytes)) => bytes.starts_with(b"."), + _ => false, + } + } else { + false + } + } + _ => false, + } +} + +#[test] +fn pattern_has_uppercase_char_simple() { + assert!(pattern_has_uppercase_char("A")); + assert!(pattern_has_uppercase_char("foo.EXE")); + + assert!(!pattern_has_uppercase_char("a")); + assert!(!pattern_has_uppercase_char("foo.exe123")); +} + +#[test] +fn pattern_has_uppercase_char_advanced() { + assert!(pattern_has_uppercase_char("foo.[a-zA-Z]")); + + assert!(!pattern_has_uppercase_char(r"\Acargo")); + assert!(!pattern_has_uppercase_char(r"carg\x6F")); +} + +#[test] +fn matches_strings_with_leading_dot_simple() { + assert!(pattern_matches_strings_with_leading_dot("^\\.gitignore")); + + assert!(!pattern_matches_strings_with_leading_dot("^.gitignore")); + assert!(!pattern_matches_strings_with_leading_dot("\\.gitignore")); + assert!(!pattern_matches_strings_with_leading_dot("^gitignore")); +} + +#[test] +fn pattern_has_uppercase_char_inline_flags() { + for pattern in [ + "(?i)foo", + "(?i:foo)", + "(?i)[a-z]", + "(?i)[a-z&&[^q]]", + "(?i)é", + ] { + assert!(!pattern_has_uppercase_char(pattern), "{pattern}"); + } + for pattern in ["(?i)Foo", "(?i)[A-Z]", r"(?i)\x46oo", "(?i)É"] { + assert!(pattern_has_uppercase_char(pattern), "{pattern}"); + } +} diff --git a/userland/fd/src/sanitize.rs b/userland/fd/src/sanitize.rs new file mode 100644 index 0000000..53f289e --- /dev/null +++ b/userland/fd/src/sanitize.rs @@ -0,0 +1,176 @@ +//! TTY-output sanitization to prevent terminal escape injection via filenames. + +use std::fmt::{Display, Formatter, Write}; + +/// True for any char that is neither printable nor permitted whitespace (only HT). +/// Covers C0/C1/DEL, bidi overrides, zero-width and format chars, and tag chars. +#[inline] +fn needs_escape(c: char) -> bool { + if c == '\t' { + return false; + } + c.is_control() + || matches!(c, + '\u{00AD}' // soft hyphen (invisible) + | '\u{180E}' // Mongolian vowel separator + | '\u{200B}'..='\u{200F}' // zero-width + LRM/RLM + | '\u{202A}'..='\u{202E}' // bidi embedding/override + | '\u{2060}'..='\u{206F}' // word joiner, invisibles, deprecated formats + | '\u{FEFF}' // BOM / zero-width no-break space + | '\u{FFF9}'..='\u{FFFB}' // interlinear annotation + | '\u{E0000}'..='\u{E007F}' // language tags + ) +} + +/// Write the sanitized contents of `raw` to `f`. +pub fn write_sanitized(f: &mut std::fmt::Formatter<'_>, raw: &str) -> std::fmt::Result { + // Would it be faster to do a pass to see if we don't need an escape and write + // the whole string first? Maybe with a faster check just for non-control ASCII? + for c in raw.chars() { + if needs_escape(c) { + let v = c as u32; + if v <= 0xFF { + write!(f, "\\x{v:02X}")?; + } else { + write!(f, "\\u{{{v:04X}}}")?; + } + } else { + f.write_char(c)?; + } + } + Ok(()) +} + +// TODO: add something to sanitize paths directly instead of as strings. + +/// A wrapper type that sanitizes the output to Display +pub(crate) struct SanitizedStr<'a> { + raw: &'a str, + /// If true, the content is safe to output without sanitizing, + /// either because it is trusted, or because it isn't going to a terminal. + is_safe: bool, +} + +impl<'a> Display for SanitizedStr<'a> { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + if self.is_safe { + // We don't need to sanitize anything, just forward + self.raw.fmt(f) + } else { + write_sanitized(f, self.raw) + } + } +} + +/// Sanitize for terminal output only; raw bytes pass through on pipes/files. +pub fn sanitize_for_term(raw: &str, is_terminal: bool) -> SanitizedStr<'_> { + SanitizedStr { + raw, + is_safe: !is_terminal, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn sanitize_string(s: &str) -> String { + SanitizedStr { + raw: s, + is_safe: false, + } + .to_string() + } + + #[test] + fn preserves_safe_content() { + for s in [ + "hello.txt", + "résumé.pdf", + "文档.txt", + "🦀.rs", + "a\tb", + "a\u{FFFD}b", + ] { + assert_eq!(sanitize_string(s), s); + } + } + + #[test] + fn strips_osc52_clipboard_payload() { + let attack = "innocent\x1b]52;c;cHduZWQ=\x1b\\.txt"; + let safe = sanitize_string(attack); + assert!(!safe.contains('\x1b')); + assert_eq!(safe, "innocent\\x1B]52;c;cHduZWQ=\\x1B\\.txt"); + } + + #[test] + fn strips_cr_output_forgery() { + assert_eq!(sanitize_string("A\rFAKE OUTPUT"), "A\\x0DFAKE OUTPUT"); + } + + #[test] + fn strips_osc8_hyperlink_injection() { + let attack = "phish\x1b]8;;https:evil.example\x1b\\phony.txt"; + assert!(!sanitize_string(attack).contains('\x1b')); + } + + #[test] + fn strips_del() { + assert_eq!(sanitize_string("a\x7fb"), "a\\x7Fb"); + } + + #[test] + fn strips_bel_and_null() { + assert_eq!(sanitize_string("a\x07b"), "a\\x07b"); + assert_eq!(sanitize_string("a\0b"), "a\\x00b"); + } + + #[test] + fn escape_preserves_information() { + let s = "name\x1bX\x07Y.txt"; + assert_eq!(sanitize_string(s), "name\\x1BX\\x07Y.txt"); + } + + #[test] + fn strips_c1_csi_and_osc_initiators() { + // U+009B is CSI, U+009D is OSC; dangerous on 8-bit-control terminals. + assert_eq!(sanitize_string("\u{9b}31m"), "\\x9B31m"); + assert_eq!(sanitize_string("\u{9d}0;pwned\u{9c}"), "\\x9D0;pwned\\x9C"); + } + + #[test] + fn strips_bidi_overrides_and_zero_width() { + // Trojan-Source style RLO/LRO that flip rendered order of filename text. + assert_eq!( + sanitize_string("safe\u{202E}fil\u{202D}gnp.exe"), + "safe\\u{202E}fil\\u{202D}gnp.exe" + ); + // Zero-width space and BOM are also format chars used to disguise filenames. + assert_eq!(sanitize_string("a\u{200B}b"), "a\\u{200B}b"); + assert_eq!(sanitize_string("\u{FEFF}name"), "\\u{FEFF}name"); + } + + #[test] + fn keeps_legitimate_unicode_features() { + // Variation selectors (U+FE0F, U+E0100..) modify preceding glyphs in CJK/emoji + // and are legitimate in filenames. Private-use chars are used by icon fonts. + for s in [ + "heart\u{2764}\u{FE0F}.txt", + "icon\u{E000}.cfg", + "cjk\u{6F22}\u{E0101}.txt", + ] { + assert_eq!(sanitize_string(s), s); + } + } + + #[test] + fn maybe_sanitize_passthrough_when_not_terminal() { + let attack = "x\x1by"; + // Pipe context: bytes pass through unchanged (zero-copy). + let out = sanitize_for_term(attack, false).to_string(); + assert_eq!(out, attack); + // TTY context: escapes apply. + assert_eq!(sanitize_for_term(attack, true).to_string(), "x\\x1By"); + } +} diff --git a/userland/fd/src/walk.rs b/userland/fd/src/walk.rs new file mode 100644 index 0000000..b047c85 --- /dev/null +++ b/userland/fd/src/walk.rs @@ -0,0 +1,792 @@ +use std::borrow::Cow; +use std::ffi::OsStr; +use std::io::{self, Write}; +use std::mem; +use std::path::PathBuf; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Mutex, MutexGuard}; +use std::thread; +use std::time::{Duration, Instant}; + +use anyhow::{Result, anyhow}; +use crossbeam_channel::{Receiver, RecvTimeoutError, SendError, Sender}; +#[cfg(not(target_os = "wasi"))] +use crossbeam_channel::bounded; +use etcetera::BaseStrategy; +use ignore::overrides::{Override, OverrideBuilder}; +use ignore::{WalkBuilder, WalkState}; +#[cfg(not(target_os = "wasi"))] +use ignore::WalkParallel; +#[cfg(target_os = "wasi")] +use crate::wasi_walk::WalkParallel; + +use crate::config::Config; +use crate::dir_entry::DirEntry; +use crate::exec; +use crate::exit_codes::{ExitCode, merge_exitcodes}; +use crate::output; +use crate::patterns::Patterns; + +/// The receiver thread can either be buffering results or directly streaming to the console. +#[derive(PartialEq)] +enum ReceiverMode { + /// Receiver is still buffering in order to sort the results, if the search finishes fast + /// enough. + Buffering, + + /// Receiver is directly printing results to the output. + Streaming, +} + +/// The Worker threads can result in a valid entry having PathBuf or an error. +#[allow(clippy::large_enum_variant)] +#[derive(Debug)] +pub enum WorkerResult { + // Errors should be rare, so it's probably better to allow large_enum_variant than + // to box the Entry variant + Entry(DirEntry), + Error(ignore::Error), +} + +/// A batch of WorkerResults to send over a channel. +#[derive(Clone)] +struct Batch { + items: Arc>>>, +} + +impl Batch { + fn new() -> Self { + Self { + items: Arc::new(Mutex::new(Some(vec![]))), + } + } + + fn lock(&self) -> MutexGuard<'_, Option>> { + self.items.lock().unwrap() + } +} + +impl IntoIterator for Batch { + type Item = WorkerResult; + type IntoIter = std::vec::IntoIter; + + fn into_iter(self) -> Self::IntoIter { + self.lock().take().unwrap().into_iter() + } +} + +/// Wrapper that sends batches of items at once over a channel. +struct BatchSender { + batch: Batch, + tx: Sender, + limit: usize, +} + +impl BatchSender { + fn new(tx: Sender, limit: usize) -> Self { + Self { + batch: Batch::new(), + tx, + limit, + } + } + + /// Check if we need to flush a batch. + fn needs_flush(&self, batch: Option<&Vec>) -> bool { + match batch { + // Limit the batch size to provide some backpressure + Some(vec) => vec.len() >= self.limit, + // Batch was already taken by the receiver, so make a new one + None => true, + } + } + + /// Add an item to a batch. + fn send(&mut self, item: WorkerResult) -> Result<(), SendError<()>> { + let mut batch = self.batch.lock(); + + if self.needs_flush(batch.as_ref()) { + drop(batch); + self.batch = Batch::new(); + batch = self.batch.lock(); + } + + let items = batch.as_mut().unwrap(); + items.push(item); + + if items.len() == 1 { + // New batch, send it over the channel + self.tx + .send(self.batch.clone()) + .map_err(|_| SendError(()))?; + } + + Ok(()) + } +} + +/// Maximum size of the output buffer before flushing results to the console +const MAX_BUFFER_LENGTH: usize = 1000; +/// Default duration until output buffering switches to streaming. +const DEFAULT_MAX_BUFFER_TIME: Duration = Duration::from_millis(100); + +/// Wrapper for the receiver thread's buffering behavior. +struct ReceiverBuffer<'a, W> { + /// The configuration. + config: &'a Config, + /// For shutting down the senders. + quit_flag: &'a AtomicBool, + /// The ^C notifier. + interrupt_flag: &'a AtomicBool, + /// Receiver for worker results. + rx: Receiver, + /// Standard output. + stdout: W, + /// The current buffer mode. + mode: ReceiverMode, + /// The deadline to switch to streaming mode. + deadline: Instant, + /// The buffer of quickly received paths. + buffer: Vec, + /// Result count. + num_results: usize, +} + +impl<'a, W: Write> ReceiverBuffer<'a, W> { + /// Create a new receiver buffer. + fn new(state: &'a WorkerState, rx: Receiver, stdout: W) -> Self { + let config = &state.config; + let quit_flag = state.quit_flag.as_ref(); + let interrupt_flag = state.interrupt_flag.as_ref(); + let max_buffer_time = config.max_buffer_time.unwrap_or(DEFAULT_MAX_BUFFER_TIME); + let deadline = Instant::now() + max_buffer_time; + + Self { + config, + quit_flag, + interrupt_flag, + rx, + stdout, + mode: ReceiverMode::Buffering, + deadline, + buffer: Vec::with_capacity(MAX_BUFFER_LENGTH), + num_results: 0, + } + } + + /// Process results until finished. + fn process(&mut self) -> ExitCode { + loop { + if let Err(ec) = self.poll() { + self.quit_flag.store(true, Ordering::Relaxed); + return ec; + } + } + } + + /// Receive the next worker result. + fn recv(&self) -> Result { + match self.mode { + ReceiverMode::Buffering => { + // Wait at most until we should switch to streaming + self.rx.recv_deadline(self.deadline) + } + ReceiverMode::Streaming => { + // Wait however long it takes for a result + Ok(self.rx.recv()?) + } + } + } + + /// Wait for a result or state change. + fn poll(&mut self) -> Result<(), ExitCode> { + match self.recv() { + Ok(batch) => { + for result in batch { + match result { + WorkerResult::Entry(dir_entry) => { + if self.config.quiet { + return Err(ExitCode::HasResults(true)); + } + + match self.mode { + ReceiverMode::Buffering => { + self.buffer.push(dir_entry); + if self.buffer.len() > MAX_BUFFER_LENGTH { + self.stream()?; + } + } + ReceiverMode::Streaming => { + self.print(&dir_entry)?; + } + } + + self.num_results += 1; + if let Some(max_results) = self.config.max_results + && self.num_results >= max_results + { + return self.stop(); + } + } + WorkerResult::Error(err) => { + if self.config.show_filesystem_errors { + print_error!("{}", err); + } + } + } + } + + // If we don't have another batch ready, flush before waiting + if self.mode == ReceiverMode::Streaming && self.rx.is_empty() { + self.flush()?; + } + } + Err(RecvTimeoutError::Timeout) => { + self.stream()?; + } + Err(RecvTimeoutError::Disconnected) => { + return self.stop(); + } + } + + Ok(()) + } + + /// Output a path. + fn print(&mut self, entry: &DirEntry) -> Result<(), ExitCode> { + if let Err(e) = output::print_entry(&mut self.stdout, entry, self.config) + && e.kind() != ::std::io::ErrorKind::BrokenPipe + { + print_error!("Could not write to output: {}", e); + return Err(ExitCode::GeneralError); + } + + if self.interrupt_flag.load(Ordering::Relaxed) { + // Ignore any errors on flush, because we're about to exit anyway + let _ = self.flush(); + return Err(ExitCode::KilledBySigint); + } + + Ok(()) + } + + /// Switch ourselves into streaming mode. + fn stream(&mut self) -> Result<(), ExitCode> { + self.mode = ReceiverMode::Streaming; + + let buffer = mem::take(&mut self.buffer); + for path in buffer { + self.print(&path)?; + } + + self.flush() + } + + /// Stop looping. + fn stop(&mut self) -> Result<(), ExitCode> { + if self.mode == ReceiverMode::Buffering { + self.buffer.sort(); + self.stream()?; + } + + if self.config.quiet { + Err(ExitCode::HasResults(self.num_results > 0)) + } else { + Err(ExitCode::Success) + } + } + + /// Flush stdout if necessary. + fn flush(&mut self) -> Result<(), ExitCode> { + if self.stdout.flush().is_err() { + // Probably a broken pipe. Exit gracefully. + return Err(ExitCode::GeneralError); + } + Ok(()) + } +} + +/// State shared by the sender and receiver threads. +struct WorkerState { + /// The search patterns. + patterns: Patterns, + /// The command line configuration. + config: Config, + /// Flag for cleanly shutting down the parallel walk + quit_flag: Arc, + /// Flag specifically for quitting due to ^C + interrupt_flag: Arc, +} + +impl WorkerState { + fn new(patterns: Patterns, config: Config) -> Self { + let quit_flag = Arc::new(AtomicBool::new(false)); + let interrupt_flag = Arc::new(AtomicBool::new(false)); + + Self { + patterns, + config, + quit_flag, + interrupt_flag, + } + } + + fn build_overrides(&self, paths: &[PathBuf]) -> Result { + let first_path = &paths[0]; + let config = &self.config; + + let mut builder = OverrideBuilder::new(first_path); + + for pattern in &config.exclude_patterns { + builder + .add(pattern) + .map_err(|e| anyhow!("Malformed exclude pattern: {}", e))?; + } + + builder + .build() + .map_err(|_| anyhow!("Mismatch in exclude patterns")) + } + + fn build_walker(&self, paths: &[PathBuf]) -> Result { + let first_path = &paths[0]; + let config = &self.config; + let overrides = self.build_overrides(paths)?; + + let mut builder = WalkBuilder::new(first_path); + builder + .hidden(config.ignore_hidden) + .ignore(config.read_fdignore) + .parents(config.read_parent_ignore && (config.read_fdignore || config.read_vcsignore)) + .git_ignore(config.read_vcsignore) + .git_global(config.read_vcsignore) + .git_exclude(config.read_vcsignore) + .require_git(config.require_git_to_read_vcsignore) + .overrides(overrides) + .follow_links(config.follow_links) + // No need to check for supported platforms, option is unavailable on unsupported ones + .same_file_system(config.one_file_system) + .max_depth(config.max_depth); + + if config.read_fdignore { + builder.add_custom_ignore_filename(".fdignore"); + } + + if config.read_global_ignore + && let Ok(basedirs) = etcetera::choose_base_strategy() + { + let global_ignore_file = basedirs.config_dir().join("fd").join("ignore"); + if global_ignore_file.is_file() { + let result = builder.add_ignore(global_ignore_file); + match result { + Some(ignore::Error::Partial(_)) => (), + Some(err) => { + print_error!("Malformed pattern in global ignore file. {}.", err); + } + None => (), + } + } + } + + for ignore_file in &config.ignore_files { + let result = builder.add_ignore(ignore_file); + match result { + Some(ignore::Error::Partial(_)) => (), + Some(err) => { + print_error!("Malformed pattern in custom ignore file. {}.", err); + } + None => (), + } + } + + for path in &paths[1..] { + builder.add(path); + } + + #[cfg(not(target_os = "wasi"))] + let walker = builder.threads(config.threads).build_parallel(); + #[cfg(target_os = "wasi")] + let walker = WalkParallel(builder.build()); + Ok(walker) + } + + /// Run the receiver work, either on this thread or a pool of background + /// threads (for --exec). + fn receive(&self, rx: Receiver) -> ExitCode { + let config = &self.config; + + // This will be set to `Some` if the `--exec` argument was supplied. + if let Some(ref cmd) = config.command { + if cmd.in_batch_mode() { + exec::batch(rx.into_iter().flatten(), cmd, config) + } else { + thread::scope(|scope| { + // Each spawned job will store its thread handle in here. + let threads = config.threads; + let mut handles = Vec::with_capacity(threads); + for _ in 0..threads { + let rx = rx.clone(); + + // Spawn a job thread that will listen for and execute inputs. + let handle = + scope.spawn(|| exec::job(rx.into_iter().flatten(), cmd, config)); + + // Push the handle of the spawned thread into the vector for later joining. + handles.push(handle); + } + let exit_codes = handles.into_iter().map(|handle| handle.join().unwrap()); + merge_exitcodes(exit_codes) + }) + } + } else { + let stdout = io::stdout().lock(); + let stdout = io::BufWriter::new(stdout); + + ReceiverBuffer::new(self, rx, stdout).process() + } + } + + /// Spawn the sender threads. + fn spawn_senders(&self, walker: WalkParallel, tx: Sender) { + walker.run(|| { + let patterns = &self.patterns; + let config = &self.config; + let quit_flag = self.quit_flag.as_ref(); + + let mut limit = 0x100; + if let Some(cmd) = &config.command + && !cmd.in_batch_mode() + && config.threads > 1 + { + // Evenly distribute work between multiple receivers + limit = 1; + } + let mut tx = BatchSender::new(tx.clone(), limit); + + Box::new(move |entry| { + if quit_flag.load(Ordering::Relaxed) { + return WalkState::Quit; + } + + if let Ok(e) = &entry { + // If the entry is a directory that contains a + // "ignore contain" file", we want to skip this + // directory. + // Check the filetype first to avoid unnecessary + // syscalls. + if e.file_type().is_some_and(|t| t.is_dir()) { + let entry_path = e.path(); + if config + .ignore_contain + .iter() + .any(|ic| entry_path.join(ic).exists()) + { + return WalkState::Skip; + } + } + if e.depth() == 0 { + // Skip the root directory entry. + return WalkState::Continue; + } + } + let entry = match entry { + Ok(e) => DirEntry::normal(e), + Err(err) => match broken_symlink_from_err(err) { + Ok(entry) => entry, + Err(err) => { + return match tx.send(WorkerResult::Error(err)) { + Ok(_) => WalkState::Continue, + Err(_) => WalkState::Quit, + }; + } + }, + }; + + if let Some(min_depth) = config.min_depth + && entry.depth().is_none_or(|d| d < min_depth) + { + return WalkState::Continue; + } + + // Check the name first, since it doesn't require metadata + let entry_path = entry.path(); + + let search_str = search_str_for_entry(entry_path, config.full_path_base.as_deref()); + + if !patterns.matches_path(&search_str) { + return WalkState::Continue; + } + + // Filter out unwanted extensions. + if let Some(ref exts_regex) = config.extensions { + if let Some(path_str) = entry_path.file_name() { + if !exts_regex.is_match(path_str.as_encoded_bytes()) { + return WalkState::Continue; + } + } else { + return WalkState::Continue; + } + } + + // Filter out unwanted file types. + if let Some(ref file_types) = config.file_types + && file_types.should_ignore(&entry) + { + return WalkState::Continue; + } + + #[cfg(unix)] + { + if let Some(ref owner_constraint) = config.owner_constraint { + if let Some(metadata) = entry.metadata() { + if !owner_constraint.matches(metadata) { + return WalkState::Continue; + } + } else { + return WalkState::Continue; + } + } + } + + // Filter out unwanted sizes if it is a file and we have been given size constraints. + if !config.size_constraints.is_empty() { + if entry_path.is_file() { + if let Some(metadata) = entry.metadata() { + let file_size = metadata.len(); + if config + .size_constraints + .iter() + .any(|sc| !sc.is_within(file_size)) + { + return WalkState::Continue; + } + } else { + return WalkState::Continue; + } + } else { + return WalkState::Continue; + } + } + + // Filter out unwanted modification times + if !config.time_constraints.is_empty() { + let mut matched = false; + if let Some(metadata) = entry.metadata() + && let Ok(modified) = metadata.modified() + { + matched = config + .time_constraints + .iter() + .all(|tf| tf.applies_to(&modified)); + } + if !matched { + return WalkState::Continue; + } + } + + if config.is_printing() + && let Some(ls_colors) = &config.ls_colors + { + // Compute colors in parallel + entry.style(ls_colors); + } + + let send_result = tx.send(WorkerResult::Entry(entry)); + + if send_result.is_err() { + return WalkState::Quit; + } + + // Apply pruning. + if config.prune { + return WalkState::Skip; + } + + WalkState::Continue + }) + }); + } + + /// Perform the recursive scan. + fn scan(&self, paths: &[PathBuf]) -> Result { + #[cfg(not(target_os = "wasi"))] + let config = &self.config; + let walker = self.build_walker(paths)?; + + #[cfg(not(target_os = "wasi"))] + if config.ls_colors.is_some() && config.is_printing() { + let quit_flag = Arc::clone(&self.quit_flag); + let interrupt_flag = Arc::clone(&self.interrupt_flag); + + ctrlc::set_handler(move || { + quit_flag.store(true, Ordering::Relaxed); + + if interrupt_flag.fetch_or(true, Ordering::Relaxed) { + // Ctrl-C has been pressed twice, exit NOW + ExitCode::KilledBySigint.exit(); + } + }) + .unwrap(); + } + + #[cfg(not(target_os = "wasi"))] + let (tx, rx) = bounded(2 * config.threads); + #[cfg(target_os = "wasi")] + let (tx, rx) = crossbeam_channel::unbounded(); + + #[cfg(target_os = "wasi")] + let exit_code = { + self.spawn_senders(walker, tx); + self.receive(rx) + }; + #[cfg(not(target_os = "wasi"))] + let exit_code = thread::scope(|scope| { + // Spawn the receiver thread(s) + let receiver = scope.spawn(|| self.receive(rx)); + + // Spawn the sender threads. + self.spawn_senders(walker, tx); + + receiver.join().unwrap() + }); + + if self.interrupt_flag.load(Ordering::Relaxed) { + Ok(ExitCode::KilledBySigint) + } else { + Ok(exit_code) + } + } +} + +/// If `err` is for a broken symlink, create a broken symlink DirEntry, +/// otherwise, return the original error. +fn broken_symlink_from_err(mut err: ignore::Error) -> Result { + use ignore::Error::*; + let mut depth = None; + let mut path = None; + let mut current = &mut err; + // Loop through the items in the chain, and extract the path and depth as we find them. + // Then once we get to the (terminal) io error, check that it is for a broken symlink. + loop { + match current { + WithPath { err, path: p } => { + path = Some(p); + current = err; + } + WithDepth { err, depth: d } => { + depth = Some(*d); + current = err; + } + Io(e) if e.kind() == io::ErrorKind::NotFound => { + if let Some(path) = path + && path + .symlink_metadata() + .ok() + .is_some_and(|m| m.file_type().is_symlink()) + { + // Since we no longer need the original error, we can take the PathBuf of the + // path out of it. + return Ok(DirEntry::broken_symlink(mem::take(path), depth)); + } else { + // This is pretty unlikely. + return Err(err); + } + } + // We don't care about Loop because that won't happen + // for broken symlinks, and there are two paths involved + // and we don't care about WithLineNumber because that only applies + // to errors while parsing ignore files. + _ => return Err(err), + } + } +} + +fn search_str_for_entry<'a>( + entry_path: &'a std::path::Path, + full_path_base: Option<&std::path::Path>, +) -> Cow<'a, OsStr> { + if let Some(cwd) = full_path_base { + // If full_path_base is some, that means that we need to return + // the absolute path + if entry_path.is_absolute() { + return Cow::Borrowed(entry_path.as_os_str()); + } + let path = entry_path.strip_prefix(".").unwrap_or(entry_path); + Cow::Owned(cwd.join(path).into()) + } else { + match entry_path.file_name() { + Some(filename) => Cow::Borrowed(filename), + None => unreachable!( + "Encountered file system entry without a file name. This should only \ + happen for paths like 'foo/bar/..' or '/' which are not supposed to \ + appear in a file system traversal." + ), + } + } +} + +/// Recursively scan the given search path for files / pathnames matching the patterns. +/// +/// If the `--exec` argument was supplied, this will create a thread pool for executing +/// jobs in parallel from a given command line and the discovered paths. Otherwise, each +/// path will simply be written to standard output. +pub fn scan(paths: &[PathBuf], patterns: Patterns, config: Config) -> Result { + WorkerState::new(patterns, config).scan(paths) +} + +#[cfg(test)] +mod tests { + use super::search_str_for_entry; + use std::path::{Path, PathBuf}; + + #[test] + fn search_str_for_entry_with_relative_path() { + let full_path_base = Some(Path::new("/home/user")); + assert_eq!( + search_str_for_entry(Path::new("foo/bar"), full_path_base), + PathBuf::from("/home/user/foo/bar") + ); + } + + #[test] + fn search_str_for_entry_strips_dot_prefix() { + let full_path_base = Some(Path::new("/home/user")); + assert_eq!( + search_str_for_entry(Path::new("./foo/bar"), full_path_base), + PathBuf::from("/home/user/foo/bar") + ); + } + + #[test] + fn search_str_for_entry_with_absolute_path() { + let full_path_base = Some(Path::new("/home/user")); + assert_eq!( + search_str_for_entry(Path::new("/absolute/path"), full_path_base), + PathBuf::from("/absolute/path") + ); + } + + #[test] + fn search_str_no_base_dir() { + assert_eq!( + search_str_for_entry(Path::new("./foo/bar"), None), + PathBuf::from("bar") + ); + } + + #[test] + fn search_str_no_base_dir_with_plain_relative_path() { + assert_eq!( + search_str_for_entry(Path::new("foo/bar"), None), + PathBuf::from("bar") + ); + } + + #[test] + fn search_str_no_base_dir_with_file_in_current_dir() { + assert_eq!( + search_str_for_entry(Path::new("foo"), None), + PathBuf::from("foo") + ); + } +} diff --git a/userland/fd/src/wasi_walk.rs b/userland/fd/src/wasi_walk.rs new file mode 100644 index 0000000..02d21d0 --- /dev/null +++ b/userland/fd/src/wasi_walk.rs @@ -0,0 +1,21 @@ +//! Run fd's visitor on the single-threaded ignore walker. +use ignore::{DirEntry, Error, Walk, WalkState}; +pub struct WalkParallel(pub Walk); +impl WalkParallel { + pub fn run<'a>(self, factory: impl FnOnce() -> Box) -> WalkState + 'a>) { + let mut visit = factory(); + let mut skip: Option = None; + for entry in self.0 { + if let Ok(e) = &entry { + if skip.as_ref().is_some_and(|p| e.path().starts_with(p)) { continue; } + skip = None; + } + let path = entry.as_ref().ok().map(|e| e.path().to_owned()); + match visit(entry) { + WalkState::Quit => break, + WalkState::Skip => skip = path, + WalkState::Continue => (), + } + } + } +} diff --git a/userland/fd/tests/testenv/mod.rs b/userland/fd/tests/testenv/mod.rs new file mode 100644 index 0000000..ef564da --- /dev/null +++ b/userland/fd/tests/testenv/mod.rs @@ -0,0 +1,360 @@ +use std::env; +use std::fs; +use std::io::{self, Write}; +#[cfg(unix)] +use std::os::unix; +#[cfg(windows)] +use std::os::windows; +use std::path::{Path, PathBuf}; +use std::process; + +use tempfile::TempDir; + +/// Environment for the integration tests. +pub struct TestEnv { + /// Temporary working directory. + temp_dir: TempDir, + + /// Path to the *fd* executable. + fd_exe: PathBuf, + + /// Normalize each line by sorting the whitespace-separated words + normalize_line: bool, + + /// Temporary directory for storing test config (global ignore file) + config_dir: Option, +} + +/// Create the working directory and the test files. +fn create_working_directory( + directories: &[&'static str], + files: &[&'static str], +) -> Result { + let temp_dir = tempfile::Builder::new().prefix("fd-tests").tempdir()?; + + { + let root = temp_dir.path(); + + // Pretend that this is a Git repository in order for `.gitignore` files to be respected + fs::create_dir_all(root.join(".git"))?; + + for directory in directories { + fs::create_dir_all(root.join(directory))?; + } + + for file in files { + fs::File::create(root.join(file))?; + } + + #[cfg(unix)] + unix::fs::symlink(root.join("one/two"), root.join("symlink"))?; + + // Note: creating symlinks on Windows requires the `SeCreateSymbolicLinkPrivilege` which + // is by default only granted for administrators. + #[cfg(windows)] + windows::fs::symlink_dir(root.join("one/two"), root.join("symlink"))?; + + fs::File::create(root.join(".fdignore"))?.write_all(b"fdignored.foo")?; + + fs::File::create(root.join(".gitignore"))?.write_all(b"gitignored.foo")?; + } + + Ok(temp_dir) +} + +fn create_config_directory_with_global_ignore(ignore_file_content: &str) -> io::Result { + let config_dir = tempfile::Builder::new().prefix("fd-config").tempdir()?; + let fd_dir = config_dir.path().join("fd"); + fs::create_dir(&fd_dir)?; + let mut ignore_file = fs::File::create(fd_dir.join("ignore"))?; + ignore_file.write_all(ignore_file_content.as_bytes())?; + + Ok(config_dir) +} + +/// Find the *fd* executable. +fn find_fd_exe() -> PathBuf { + // Read the location of the fd executable from the environment + PathBuf::from(env::var("CARGO_BIN_EXE_fd").unwrap_or(env!("CARGO_BIN_EXE_fd").to_string())) +} + +/// Format an error message for when *fd* did not exit successfully. +fn format_exit_error(args: &[&str], output: &process::Output) -> String { + format!( + "`fd {}` did not exit successfully.\nstdout:\n---\n{}---\nstderr:\n---\n{}---", + args.join(" "), + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ) +} + +/// Format an error message for when the output of *fd* did not match the expected output. +fn format_output_error(args: &[&str], expected: &str, actual: &str) -> String { + // Generate diff text. + let diff_text = diff::lines(expected, actual) + .into_iter() + .map(|diff| match diff { + diff::Result::Left(l) => format!("-{l}"), + diff::Result::Both(l, _) => format!(" {l}"), + diff::Result::Right(r) => format!("+{r}"), + }) + .collect::>() + .join("\n"); + + format!( + concat!( + "`fd {}` did not produce the expected output.\n", + "Showing diff between expected and actual:\n{}\n" + ), + args.join(" "), + diff_text + ) +} + +/// Normalize the output for comparison. +fn normalize_output(s: &str, trim_start: bool, normalize_line: bool) -> String { + // Split into lines and normalize separators. + let mut lines = s + .replace('\0', "NULL\n") + .lines() + .map(|line| { + let line = if trim_start { line.trim_start() } else { line }; + let line = line.replace('/', std::path::MAIN_SEPARATOR_STR); + if normalize_line { + let mut words: Vec<_> = line.split_whitespace().collect(); + words.sort_unstable(); + return words.join(" "); + } + line + }) + .collect::>(); + + lines.sort(); + lines.join("\n") +} + +/// Trim whitespace from the beginning of each line. +fn trim_lines(s: &str) -> String { + s.lines() + .map(|line| line.trim_start()) + .fold(String::new(), |mut str, line| { + str.push_str(line); + str.push('\n'); + str + }) +} + +impl TestEnv { + pub fn new(directories: &[&'static str], files: &[&'static str]) -> TestEnv { + let temp_dir = create_working_directory(directories, files).expect("working directory"); + let fd_exe = find_fd_exe(); + + TestEnv { + temp_dir, + fd_exe, + normalize_line: false, + config_dir: None, + } + } + + pub fn normalize_line(self, normalize: bool) -> TestEnv { + TestEnv { + temp_dir: self.temp_dir, + fd_exe: self.fd_exe, + normalize_line: normalize, + config_dir: self.config_dir, + } + } + + pub fn global_ignore_file(self, content: &str) -> TestEnv { + let config_dir = + create_config_directory_with_global_ignore(content).expect("config directory"); + TestEnv { + config_dir: Some(config_dir), + ..self + } + } + + /// Create a broken symlink at the given path in the temp_dir. + pub fn create_broken_symlink>( + &mut self, + link_path: P, + ) -> Result { + let root = self.test_root(); + let broken_symlink_link = root.join(link_path); + { + let temp_target_dir = tempfile::Builder::new() + .prefix("fd-tests-broken-symlink") + .tempdir()?; + let broken_symlink_target = temp_target_dir.path().join("broken_symlink_target"); + fs::File::create(&broken_symlink_target)?; + #[cfg(unix)] + unix::fs::symlink(&broken_symlink_target, &broken_symlink_link)?; + #[cfg(windows)] + windows::fs::symlink_file(&broken_symlink_target, &broken_symlink_link)?; + } + Ok(broken_symlink_link) + } + + pub fn create_symlink>( + &mut self, + target: P, + link_path: P, + ) -> io::Result { + let root = self.test_root(); + let target_path = root.join(target); + let link_path = root.join(link_path); + #[cfg(unix)] + unix::fs::symlink(&target_path, &link_path)?; + #[cfg(windows)] + windows::fs::symlink_file(&target_path, &link_path)?; + + Ok(link_path) + } + + /// Get the root directory for the tests. + pub fn test_root(&self) -> PathBuf { + self.temp_dir.path().to_path_buf() + } + + /// Get the path of the fd executable. + #[cfg_attr(windows, allow(unused))] + pub fn test_exe(&self) -> &PathBuf { + &self.fd_exe + } + + /// Get the root directory of the file system. + pub fn system_root(&self) -> PathBuf { + let mut components = self.temp_dir.path().components(); + PathBuf::from(components.next().expect("root directory").as_os_str()) + } + + /// Assert that calling *fd* in the specified path under the root working directory, + /// and with the specified arguments produces the expected output. + pub fn assert_success_and_get_output>( + &self, + path: P, + args: &[&str], + ) -> process::Output { + // Run *fd*. + let output = self.run_command(path.as_ref(), args); + + // Check for exit status. + if !output.status.success() { + panic!("{}", format_exit_error(args, &output)); + } + + output + } + + pub fn assert_success_and_get_normalized_output>( + &self, + path: P, + args: &[&str], + ) -> String { + let output = self.assert_success_and_get_output(path, args); + normalize_output( + &String::from_utf8_lossy(&output.stdout), + false, + self.normalize_line, + ) + } + + /// Assert that calling *fd* with the specified arguments produces the expected output. + pub fn assert_output(&self, args: &[&str], expected: &str) { + self.assert_output_subdirectory(".", args, expected) + } + + /// Similar to assert_output, but able to handle non-utf8 output + #[cfg(all(unix, not(target_os = "macos")))] + pub fn assert_output_raw(&self, args: &[&str], expected: &[u8]) { + let output = self.assert_success_and_get_output(".", args); + + assert_eq!(expected, &output.stdout[..]); + } + + /// Assert that calling *fd* in the specified path under the root working directory, + /// and with the specified arguments produces the expected output. + pub fn assert_output_subdirectory>( + &self, + path: P, + args: &[&str], + expected: &str, + ) { + // Normalize both expected and actual output. + let expected = normalize_output(expected, true, self.normalize_line); + let actual = self.assert_success_and_get_normalized_output(path, args); + + // Compare actual output to expected output. + if expected != actual { + panic!("{}", format_output_error(args, &expected, &actual)); + } + } + + /// Assert that calling *fd* with the specified arguments produces the expected error, + /// and does not succeed. + pub fn assert_failure_with_error(&self, args: &[&str], expected: &str) { + let status = self.assert_error_subdirectory(".", args, Some(expected)); + if status.success() { + panic!("error '{expected}' did not occur."); + } + } + + /// Assert that calling *fd* with the specified arguments does not succeed. + pub fn assert_failure(&self, args: &[&str]) { + let status = self.assert_error_subdirectory(".", args, None); + if status.success() { + panic!("Failure did not occur as expected."); + } + } + + /// Assert that calling *fd* with the specified arguments produces the expected error. + pub fn assert_error(&self, args: &[&str], expected: &str) -> process::ExitStatus { + self.assert_error_subdirectory(".", args, Some(expected)) + } + + fn run_command(&self, path: &Path, args: &[&str]) -> process::Output { + // Setup *fd* command. + let mut cmd = process::Command::new(&self.fd_exe); + cmd.current_dir(self.temp_dir.path().join(path)); + if let Some(config_dir) = &self.config_dir { + cmd.env("XDG_CONFIG_HOME", config_dir.path()); + } else { + cmd.arg("--no-global-ignore-file"); + } + // Make sure LS_COLORS is unset to ensure consistent + // color output + cmd.env("LS_COLORS", ""); + cmd.args(args); + + // Run *fd*. + cmd.output().expect("fd output") + } + + /// Assert that calling *fd* in the specified path under the root working directory, + /// and with the specified arguments produces an error with the expected message. + fn assert_error_subdirectory>( + &self, + path: P, + args: &[&str], + expected: Option<&str>, + ) -> process::ExitStatus { + let output = self.run_command(path.as_ref(), args); + + if let Some(expected) = expected { + // Normalize both expected and actual output. + let expected_error = trim_lines(expected); + let actual_err = trim_lines(&String::from_utf8_lossy(&output.stderr)); + + // Compare actual output to expected output. + if !actual_err.trim_start().starts_with(&expected_error) { + panic!( + "{}", + format_output_error(args, &expected_error, &actual_err) + ); + } + } + + output.status + } +} diff --git a/userland/fd/tests/tests.rs b/userland/fd/tests/tests.rs new file mode 100644 index 0000000..5503a56 --- /dev/null +++ b/userland/fd/tests/tests.rs @@ -0,0 +1,3168 @@ +mod testenv; + +#[cfg(unix)] +use nix::unistd::{Gid, Group, Uid, User}; +use std::fs; +use std::io::Write; +use std::path::Path; +use std::time::{Duration, SystemTime}; +use test_case::test_case; + +use jiff::Timestamp; +use normpath::PathExt; +use regex::escape; + +use crate::testenv::TestEnv; + +static DEFAULT_DIRS: &[&str] = &["one/two/three", "one/two/three/directory_foo"]; + +static DEFAULT_FILES: &[&str] = &[ + "a.foo", + "one/b.foo", + "one/two/c.foo", + "one/two/C.Foo2", + "one/two/three/d.foo", + "fdignored.foo", + "gitignored.foo", + ".hidden.foo", + "e1 e2", +]; + +#[allow(clippy::let_and_return)] +fn get_absolute_root_path(env: &TestEnv) -> String { + let path = env + .test_root() + .normalize() + .expect("absolute path") + .as_path() + .to_str() + .expect("string") + .to_string(); + + #[cfg(windows)] + let path = path.trim_start_matches(r"\\?\").to_string(); + + path +} + +#[cfg(test)] +fn get_test_env_with_abs_path(dirs: &[&'static str], files: &[&'static str]) -> (TestEnv, String) { + let env = TestEnv::new(dirs, files); + let root_path = get_absolute_root_path(&env); + (env, root_path) +} + +#[cfg(test)] +fn create_file_with_size>(path: P, size_in_bytes: usize) { + let content = "#".repeat(size_in_bytes); + let mut f = fs::File::create::

(path).unwrap(); + f.write_all(content.as_bytes()).unwrap(); +} + +/// Simple test +#[test] +fn test_simple() { + let te = TestEnv::new(DEFAULT_DIRS, DEFAULT_FILES); + + te.assert_output(&["a.foo"], "a.foo"); + te.assert_output(&["b.foo"], "one/b.foo"); + te.assert_output(&["d.foo"], "one/two/three/d.foo"); + + te.assert_output( + &["foo"], + "a.foo + one/b.foo + one/two/c.foo + one/two/C.Foo2 + one/two/three/d.foo + one/two/three/directory_foo/", + ); +} + +static AND_EXTRA_FILES: &[&str] = &[ + "a.foo", + "one/b.foo", + "one/two/c.foo", + "one/two/C.Foo2", + "one/two/three/baz-quux", + "one/two/three/Baz-Quux2", + "one/two/three/d.foo", + "fdignored.foo", + "gitignored.foo", + ".hidden.foo", + "A-B.jpg", + "A-C.png", + "B-A.png", + "B-C.png", + "C-A.jpg", + "C-B.png", + "e1 e2", +]; + +/// AND test +#[test] +fn test_and_basic() { + let te = TestEnv::new(DEFAULT_DIRS, AND_EXTRA_FILES); + + te.assert_output( + &["foo", "--and", "c"], + "one/two/C.Foo2 + one/two/c.foo + one/two/three/directory_foo/", + ); + + te.assert_output( + &["f", "--and", "[ad]", "--and", "[_]"], + "one/two/three/directory_foo/", + ); + + te.assert_output( + &["f", "--and", "[ad]", "--and", "[.]"], + "a.foo + one/two/three/d.foo", + ); + + te.assert_output(&["Foo", "--and", "C"], "one/two/C.Foo2"); + + te.assert_output(&["foo", "--and", "asdasdasdsadasd"], ""); +} + +#[test] +fn test_and_empty_pattern() { + let te = TestEnv::new(DEFAULT_DIRS, AND_EXTRA_FILES); + te.assert_output(&["Foo", "--and", "2", "--and", ""], "one/two/C.Foo2"); +} + +#[test] +fn test_and_bad_pattern() { + let te = TestEnv::new(DEFAULT_DIRS, AND_EXTRA_FILES); + + te.assert_failure(&["Foo", "--and", "2", "--and", "[", "--and", "C"]); + te.assert_failure(&["Foo", "--and", "[", "--and", "2", "--and", "C"]); + te.assert_failure(&["Foo", "--and", "2", "--and", "C", "--and", "["]); + te.assert_failure(&["[", "--and", "2", "--and", "C", "--and", "Foo"]); +} + +#[test] +fn test_and_pattern_starts_with_dash() { + let te = TestEnv::new(DEFAULT_DIRS, AND_EXTRA_FILES); + + te.assert_output( + &["baz", "--and", "quux"], + "one/two/three/Baz-Quux2 + one/two/three/baz-quux", + ); + te.assert_output( + &["baz", "--and", "-"], + "one/two/three/Baz-Quux2 + one/two/three/baz-quux", + ); + te.assert_output( + &["Quu", "--and", "x", "--and", "-"], + "one/two/three/Baz-Quux2", + ); +} + +#[test] +fn test_and_plus_extension() { + let te = TestEnv::new(DEFAULT_DIRS, AND_EXTRA_FILES); + + te.assert_output( + &[ + "A", + "--and", + "B", + "--extension", + "jpg", + "--extension", + "png", + ], + "A-B.jpg + B-A.png", + ); + + te.assert_output( + &[ + "A", + "--extension", + "jpg", + "--and", + "B", + "--extension", + "png", + ], + "A-B.jpg + B-A.png", + ); +} + +#[test] +fn test_and_plus_type() { + let te = TestEnv::new(DEFAULT_DIRS, AND_EXTRA_FILES); + + te.assert_output( + &["c", "--type", "d", "--and", "foo"], + "one/two/three/directory_foo/", + ); + + te.assert_output( + &["c", "--type", "f", "--and", "foo"], + "one/two/C.Foo2 + one/two/c.foo", + ); +} + +#[test] +fn test_and_plus_glob() { + let te = TestEnv::new(DEFAULT_DIRS, AND_EXTRA_FILES); + + te.assert_output(&["*foo", "--glob", "--and", "c*"], "one/two/c.foo"); +} + +#[test] +fn test_and_plus_fixed_strings() { + let te = TestEnv::new(DEFAULT_DIRS, AND_EXTRA_FILES); + + te.assert_output( + &["foo", "--fixed-strings", "--and", "c", "--and", "."], + "one/two/c.foo + one/two/C.Foo2", + ); + + te.assert_output( + &["foo", "--fixed-strings", "--and", "[c]", "--and", "."], + "", + ); + + te.assert_output( + &["Foo", "--fixed-strings", "--and", "C", "--and", "."], + "one/two/C.Foo2", + ); +} + +#[test] +fn test_and_plus_ignore_case() { + let te = TestEnv::new(DEFAULT_DIRS, AND_EXTRA_FILES); + + te.assert_output( + &["Foo", "--ignore-case", "--and", "C", "--and", "[.]"], + "one/two/C.Foo2 + one/two/c.foo", + ); +} + +#[test] +fn test_and_plus_case_sensitive() { + let te = TestEnv::new(DEFAULT_DIRS, AND_EXTRA_FILES); + + te.assert_output( + &["foo", "--case-sensitive", "--and", "c", "--and", "[.]"], + "one/two/c.foo", + ); +} + +#[test] +fn test_and_plus_full_path() { + let te = TestEnv::new(DEFAULT_DIRS, AND_EXTRA_FILES); + + te.assert_output( + &[ + "three", + "--full-path", + "--and", + "_foo", + "--and", + r"[/\\]dir", + ], + "one/two/three/directory_foo/", + ); + + te.assert_output( + &[ + "three", + "--full-path", + "--and", + r"[/\\]two", + "--and", + r"[/\\]dir", + ], + "one/two/three/directory_foo/", + ); +} + +/// Test each pattern type with an empty pattern. +#[test] +fn test_empty_pattern() { + let te = TestEnv::new(DEFAULT_DIRS, DEFAULT_FILES); + let expected = "a.foo + e1 e2 + one/ + one/b.foo + one/two/ + one/two/c.foo + one/two/C.Foo2 + one/two/three/ + one/two/three/d.foo + one/two/three/directory_foo/ + symlink"; + + te.assert_output(&["--regex"], expected); + te.assert_output(&["--fixed-strings"], expected); + te.assert_output(&["--glob"], expected); +} + +/// Test multiple directory searches +#[test] +fn test_multi_file() { + let dirs = &["test1", "test2"]; + let files = &["test1/a.foo", "test1/b.foo", "test2/a.foo"]; + let te = TestEnv::new(dirs, files); + te.assert_output( + &["a.foo", "test1", "test2"], + "test1/a.foo + test2/a.foo", + ); + + te.assert_output( + &["", "test1", "test2"], + "test1/a.foo + test2/a.foo + test1/b.foo", + ); + + te.assert_output(&["a.foo", "test1"], "test1/a.foo"); + + te.assert_output(&["b.foo", "test1", "test2"], "test1/b.foo"); +} + +/// Test search over multiple directory with missing +#[test] +fn test_multi_file_with_missing() { + let dirs = &["real"]; + let files = &["real/a.foo", "real/b.foo"]; + let te = TestEnv::new(dirs, files); + te.assert_output(&["a.foo", "real", "fake"], "real/a.foo"); + + te.assert_error( + &["a.foo", "real", "fake"], + "[fd error]: Search path \"fake\" is not a directory.", + ); + + te.assert_output( + &["", "real", "fake"], + "real/a.foo + real/b.foo", + ); + + te.assert_output( + &["", "real", "fake1", "fake2"], + "real/a.foo + real/b.foo", + ); + + te.assert_error( + &["", "real", "fake1", "fake2"], + "[fd error]: Search path \"fake1\" is not a directory. + [fd error]: Search path \"fake2\" is not a directory.", + ); + + te.assert_failure_with_error( + &["", "fake1", "fake2"], + "[fd error]: Search path \"fake1\" is not a directory. + [fd error]: Search path \"fake2\" is not a directory. + [fd error]: No valid search paths given.", + ); +} + +/// Without --full-path, a pattern containing '/' should always produce the +/// path-separator diagnostic, even if the pattern does not name an existing +/// directory. Before the fix for sharkdp/fd#1873 this only fired when the +/// pattern happened to resolve to a real directory, so the common typo of +/// pasting a full path silently returned zero matches. +#[test] +fn test_pattern_with_forward_slash_is_rejected() { + let te = TestEnv::new(DEFAULT_DIRS, DEFAULT_FILES); + + // Pattern that is NOT a real directory; old behaviour: no warning. + te.assert_failure_with_error( + &["nonexistent/path"], + "[fd error]: The search pattern 'nonexistent/path' contains a path-separation character and will not lead to any search results.", + ); + + // Pattern that IS a real directory; old behaviour: warning. Must still fire. + te.assert_failure_with_error( + &["one/two/three"], + "[fd error]: The search pattern 'one/two/three' contains a path-separation character and will not lead to any search results.", + ); +} + +/// --full-path is the user's explicit opt-in to regex-over-full-path matching, +/// so a path-separation character in the pattern is expected and must not +/// trigger the diagnostic. +/// +/// Gated off Windows: the actual match is regex-over-the-full-path, so a +/// forward-slash pattern only matches Unix-style paths. On Windows the OS +/// uses `\` and `one/two/c` (as a literal regex) does not match a real +/// entry — the behaviour this test is pinning (the diagnostic does not +/// fire) is covered by the fact that the invocation does not error. +#[test] +#[cfg(not(windows))] +fn test_pattern_with_forward_slash_allowed_with_full_path() { + let te = TestEnv::new(DEFAULT_DIRS, DEFAULT_FILES); + + te.assert_output( + &["--full-path", "one/two/c"], + "one/two/c.foo + one/two/C.Foo2", + ); +} + +/// `--and` patterns are matched against the file name exactly like the primary +/// pattern, so a path separator in any of them is the same silent "no results" +/// footgun and must trigger the same diagnostic. Regression for the sibling of +/// #1873 left unchecked by #1975 (only the positional pattern was validated). +#[test] +fn test_and_pattern_with_forward_slash_is_rejected() { + let te = TestEnv::new(DEFAULT_DIRS, DEFAULT_FILES); + + // Clean primary pattern, but a `--and` pattern carrying a path separator. + te.assert_failure_with_error( + &["foo", "--and", "nonexistent/path"], + "[fd error]: The search pattern 'nonexistent/path' contains a path-separation character and will not lead to any search results.", + ); + + // `--full-path` is the explicit opt-in and must still suppress the + // diagnostic for `--and` patterns too (the invocation runs and matches + // instead of erroring on the path separator). + #[cfg(not(windows))] + te.assert_output( + &["--full-path", "one/two/c", "--and", "two/c"], + "one/two/c.foo + one/two/C.Foo2", + ); +} + +/// Explicit root path +#[test] +fn test_explicit_root_path() { + let te = TestEnv::new(DEFAULT_DIRS, DEFAULT_FILES); + + te.assert_output( + &["foo", "one"], + "one/b.foo + one/two/c.foo + one/two/C.Foo2 + one/two/three/d.foo + one/two/three/directory_foo/", + ); + + te.assert_output( + &["foo", "one/two/three"], + "one/two/three/d.foo + one/two/three/directory_foo/", + ); + + te.assert_output_subdirectory( + "one/two/", + &["foo", "../../"], + "../../a.foo + ../../one/b.foo + ../../one/two/c.foo + ../../one/two/C.Foo2 + ../../one/two/three/d.foo + ../../one/two/three/directory_foo/", + ); + + te.assert_output_subdirectory( + "one/two/three", + &["", ".."], + "../c.foo + ../C.Foo2 + ../three/ + ../three/d.foo + ../three/directory_foo/", + ); +} + +#[test] +fn test_single_dash_root_path() { + let te = TestEnv::new(&["-"], &["-/foo"]); + + te.assert_output(&[".", "-"], "./-/foo"); +} + +/// Regex searches +#[test] +fn test_regex_searches() { + let te = TestEnv::new(DEFAULT_DIRS, DEFAULT_FILES); + + te.assert_output( + &["[a-c].foo"], + "a.foo + one/b.foo + one/two/c.foo + one/two/C.Foo2", + ); + + te.assert_output( + &["--case-sensitive", "[a-c].foo"], + "a.foo + one/b.foo + one/two/c.foo", + ); +} + +/// Smart case +#[test] +fn test_smart_case() { + let te = TestEnv::new(DEFAULT_DIRS, DEFAULT_FILES); + + te.assert_output( + &["c.foo"], + "one/two/c.foo + one/two/C.Foo2", + ); + + te.assert_output(&["C.Foo"], "one/two/C.Foo2"); + + te.assert_output(&["Foo"], "one/two/C.Foo2"); + + // Only literal uppercase chars should trigger case sensitivity. + te.assert_output( + &["\\Ac"], + "one/two/c.foo + one/two/C.Foo2", + ); + te.assert_output(&["\\AC"], "one/two/C.Foo2"); +} + +#[test] +fn test_smart_case_inline_flags_and_patterns() { + let te = TestEnv::new(&[], &["foo-bar-1", "foo-BAR-2", "FOO-BAR-3"]); + let all = "FOO-BAR-3\nfoo-BAR-2\nfoo-bar-1"; + + te.assert_output(&["foo", "--and", "bar"], all); + te.assert_output(&["(?i)foo", "--and", "bar"], all); + te.assert_output(&["foo", "--and", "(?i)bar"], all); + te.assert_output(&["(?i:foo)", "--and", "bar"], all); + te.assert_output(&["--ignore-case", "(?i)foo", "--and", "bar"], all); + te.assert_output( + &["--case-sensitive", "(?i)foo", "--and", "bar"], + "foo-bar-1", + ); + te.assert_output(&["(?i)Foo", "--and", "bar"], "foo-bar-1"); +} + +/// Case sensitivity (--case-sensitive) +#[test] +fn test_case_sensitive() { + let te = TestEnv::new(DEFAULT_DIRS, DEFAULT_FILES); + + te.assert_output(&["--case-sensitive", "c.foo"], "one/two/c.foo"); + + te.assert_output(&["--case-sensitive", "C.Foo"], "one/two/C.Foo2"); + + te.assert_output( + &["--ignore-case", "--case-sensitive", "C.Foo"], + "one/two/C.Foo2", + ); +} + +/// Case insensitivity (--ignore-case) +#[test] +fn test_case_insensitive() { + let te = TestEnv::new(DEFAULT_DIRS, DEFAULT_FILES); + + te.assert_output( + &["--ignore-case", "C.Foo"], + "one/two/c.foo + one/two/C.Foo2", + ); + + te.assert_output( + &["--case-sensitive", "--ignore-case", "C.Foo"], + "one/two/c.foo + one/two/C.Foo2", + ); +} + +/// Glob-based searches (--glob) +#[test] +fn test_glob_searches() { + let te = TestEnv::new(DEFAULT_DIRS, DEFAULT_FILES); + + te.assert_output( + &["--glob", "*.foo"], + "a.foo + one/b.foo + one/two/c.foo + one/two/three/d.foo", + ); + + te.assert_output( + &["--glob", "[a-c].foo"], + "a.foo + one/b.foo + one/two/c.foo", + ); + + te.assert_output( + &["--glob", "[a-c].foo*"], + "a.foo + one/b.foo + one/two/C.Foo2 + one/two/c.foo", + ); +} + +/// Glob-based searches (--glob) in combination with full path searches (--full-path) +#[test] +fn test_full_path_glob_searches() { + let te = TestEnv::new(DEFAULT_DIRS, DEFAULT_FILES); + + te.assert_output( + &["--glob", "--full-path", "**/one/**/*.foo"], + "one/b.foo + one/two/c.foo + one/two/three/d.foo", + ); + + te.assert_output( + &["--glob", "--full-path", "**/one/*/*.foo"], + " one/two/c.foo", + ); + + te.assert_output( + &["--glob", "--full-path", "**/one/*/*/*.foo"], + " one/two/three/d.foo", + ); +} + +#[test] +fn test_smart_case_glob_searches() { + let te = TestEnv::new(DEFAULT_DIRS, DEFAULT_FILES); + + te.assert_output( + &["--glob", "c.foo*"], + "one/two/C.Foo2 + one/two/c.foo", + ); + + te.assert_output(&["--glob", "C.Foo*"], "one/two/C.Foo2"); + te.assert_output(&["--glob", "*.Foo*"], "one/two/C.Foo2"); +} + +/// Glob-based searches (--glob) in combination with --case-sensitive +#[test] +fn test_case_sensitive_glob_searches() { + let te = TestEnv::new(DEFAULT_DIRS, DEFAULT_FILES); + + te.assert_output(&["--glob", "--case-sensitive", "c.foo*"], "one/two/c.foo"); +} + +/// Glob-based searches (--glob) in combination with --extension +#[test] +fn test_glob_searches_with_extension() { + let te = TestEnv::new(DEFAULT_DIRS, DEFAULT_FILES); + + te.assert_output( + &["--glob", "--extension", "foo2", "[a-z].*"], + "one/two/C.Foo2", + ); +} + +/// Make sure that --regex overrides --glob +#[test] +fn test_regex_overrides_glob() { + let te = TestEnv::new(DEFAULT_DIRS, DEFAULT_FILES); + + te.assert_output(&["--glob", "--regex", "Foo2$"], "one/two/C.Foo2"); +} + +/// Full path search (--full-path) +#[test] +fn test_full_path() { + let te = TestEnv::new(DEFAULT_DIRS, DEFAULT_FILES); + + let root = te.system_root(); + let prefix = escape(&root.to_string_lossy()); + + te.assert_output( + &["--full-path", &format!("^{prefix}.*three.*foo$")], + "one/two/three/d.foo + one/two/three/directory_foo/", + ); +} + +/// Hidden files (--hidden) +#[test] +fn test_hidden() { + let te = TestEnv::new(DEFAULT_DIRS, DEFAULT_FILES); + + te.assert_output( + &["--hidden", "foo"], + ".hidden.foo + a.foo + one/b.foo + one/two/c.foo + one/two/C.Foo2 + one/two/three/d.foo + one/two/three/directory_foo/", + ); +} + +/// Hidden file attribute on Windows +#[cfg(windows)] +#[test] +fn test_hidden_file_attribute() { + use std::os::windows::fs::OpenOptionsExt; + + let te = TestEnv::new(DEFAULT_DIRS, DEFAULT_FILES); + + // https://docs.microsoft.com/en-us/windows/win32/api/fileapi/nf-fileapi-setfileattributesa + const FILE_ATTRIBUTE_HIDDEN: u32 = 2; + + fs::OpenOptions::new() + .create(true) + .write(true) + .attributes(FILE_ATTRIBUTE_HIDDEN) + .open(te.test_root().join("hidden-file.txt")) + .unwrap(); + + te.assert_output(&["--hidden", "hidden-file.txt"], "hidden-file.txt"); + te.assert_output(&["hidden-file.txt"], ""); +} + +/// Ignored files (--no-ignore) +#[test] +fn test_no_ignore() { + let te = TestEnv::new(DEFAULT_DIRS, DEFAULT_FILES); + + te.assert_output( + &["--no-ignore", "foo"], + "a.foo + fdignored.foo + gitignored.foo + one/b.foo + one/two/c.foo + one/two/C.Foo2 + one/two/three/d.foo + one/two/three/directory_foo/", + ); + + te.assert_output( + &["--hidden", "--no-ignore", "foo"], + ".hidden.foo + a.foo + fdignored.foo + gitignored.foo + one/b.foo + one/two/c.foo + one/two/C.Foo2 + one/two/three/d.foo + one/two/three/directory_foo/", + ); +} + +/// .gitignore and .fdignore +#[test] +fn test_gitignore_and_fdignore() { + let files = &[ + "ignored-by-nothing", + "ignored-by-fdignore", + "ignored-by-gitignore", + "ignored-by-both", + ]; + let te = TestEnv::new(&[], files); + + fs::File::create(te.test_root().join(".fdignore")) + .unwrap() + .write_all(b"ignored-by-fdignore\nignored-by-both") + .unwrap(); + + fs::File::create(te.test_root().join(".gitignore")) + .unwrap() + .write_all(b"ignored-by-gitignore\nignored-by-both") + .unwrap(); + + te.assert_output(&["ignored"], "ignored-by-nothing"); + + te.assert_output( + &["--no-ignore-vcs", "ignored"], + "ignored-by-nothing + ignored-by-gitignore", + ); + + te.assert_output( + &["--no-ignore", "ignored"], + "ignored-by-nothing + ignored-by-fdignore + ignored-by-gitignore + ignored-by-both", + ); +} + +/// Ignore parent ignore files (--no-ignore-parent) +#[test] +fn test_no_ignore_parent() { + let dirs = &["inner"]; + let files = &[ + "inner/parent-ignored", + "inner/child-ignored", + "inner/not-ignored", + ]; + let te = TestEnv::new(dirs, files); + + // Ignore 'parent-ignored' in root + fs::File::create(te.test_root().join(".gitignore")) + .unwrap() + .write_all(b"parent-ignored") + .unwrap(); + // Ignore 'child-ignored' in inner + fs::File::create(te.test_root().join("inner/.gitignore")) + .unwrap() + .write_all(b"child-ignored") + .unwrap(); + + te.assert_output_subdirectory("inner", &[], "not-ignored"); + + te.assert_output_subdirectory( + "inner", + &["--no-ignore-parent"], + "parent-ignored + not-ignored", + ); +} + +/// Ignore parent ignore files (--no-ignore-parent) with an inner git repo +#[test] +fn test_no_ignore_parent_inner_git() { + let dirs = &["inner"]; + let files = &[ + "inner/parent-ignored", + "inner/child-ignored", + "inner/not-ignored", + ]; + let te = TestEnv::new(dirs, files); + + // Make the inner folder also appear as a git repo + fs::create_dir_all(te.test_root().join("inner/.git")).unwrap(); + + // Ignore 'parent-ignored' in root + fs::File::create(te.test_root().join(".gitignore")) + .unwrap() + .write_all(b"parent-ignored") + .unwrap(); + // Ignore 'child-ignored' in inner + fs::File::create(te.test_root().join("inner/.gitignore")) + .unwrap() + .write_all(b"child-ignored") + .unwrap(); + + te.assert_output_subdirectory( + "inner", + &[], + "not-ignored + parent-ignored", + ); + + te.assert_output_subdirectory( + "inner", + &["--no-ignore-parent"], + "not-ignored + parent-ignored", + ); +} + +/// Precedence of .fdignore files +#[test] +fn test_custom_ignore_precedence() { + let dirs = &["inner"]; + let files = &["inner/foo"]; + let te = TestEnv::new(dirs, files); + + // Ignore 'foo' via .gitignore + fs::File::create(te.test_root().join("inner/.gitignore")) + .unwrap() + .write_all(b"foo") + .unwrap(); + + // Whitelist 'foo' via .fdignore + fs::File::create(te.test_root().join(".fdignore")) + .unwrap() + .write_all(b"!foo") + .unwrap(); + + te.assert_output(&["foo"], "inner/foo"); + + te.assert_output(&["--no-ignore-vcs", "foo"], "inner/foo"); + + te.assert_output(&["--no-ignore", "foo"], "inner/foo"); +} + +/// Don't require git to respect gitignore (--no-require-git) +#[test] +fn test_respect_ignore_files() { + let te = TestEnv::new(DEFAULT_DIRS, DEFAULT_FILES); + + // Not in a git repo anymore + fs::remove_dir(te.test_root().join(".git")).unwrap(); + + // don't respect gitignore because we're not in a git repo + te.assert_output( + &["foo"], + "a.foo + gitignored.foo + one/b.foo + one/two/c.foo + one/two/C.Foo2 + one/two/three/d.foo + one/two/three/directory_foo/", + ); + + // respect gitignore because we set `--no-require-git` + te.assert_output( + &["--no-require-git", "foo"], + "a.foo + one/b.foo + one/two/c.foo + one/two/C.Foo2 + one/two/three/d.foo + one/two/three/directory_foo/", + ); + + // make sure overriding works + te.assert_output( + &["--no-require-git", "--require-git", "foo"], + "a.foo + gitignored.foo + one/b.foo + one/two/c.foo + one/two/C.Foo2 + one/two/three/d.foo + one/two/three/directory_foo/", + ); + + te.assert_output( + &["--no-require-git", "--no-ignore", "foo"], + "a.foo + gitignored.foo + fdignored.foo + one/b.foo + one/two/c.foo + one/two/C.Foo2 + one/two/three/d.foo + one/two/three/directory_foo/", + ); +} + +/// VCS ignored files (--no-ignore-vcs) +#[test] +fn test_no_ignore_vcs() { + let te = TestEnv::new(DEFAULT_DIRS, DEFAULT_FILES); + + te.assert_output( + &["--no-ignore-vcs", "foo"], + "a.foo + gitignored.foo + one/b.foo + one/two/c.foo + one/two/C.Foo2 + one/two/three/d.foo + one/two/three/directory_foo/", + ); +} + +/// Test that --no-ignore-vcs still respects .fdignored in parent directory +#[test] +fn test_no_ignore_vcs_child_dir() { + let te = TestEnv::new( + &["inner"], + &["inner/fdignored.foo", "inner/foo", "inner/gitignored.foo"], + ); + + te.assert_output_subdirectory( + "inner", + &["--no-ignore-vcs", "foo"], + "foo + gitignored.foo", + ); +} + +/// Custom ignore files (--ignore-file) +#[test] +fn test_custom_ignore_files() { + let te = TestEnv::new(DEFAULT_DIRS, DEFAULT_FILES); + + // Ignore 'C.Foo2' and everything in 'three'. + fs::File::create(te.test_root().join("custom.ignore")) + .unwrap() + .write_all(b"C.Foo2\nthree") + .unwrap(); + + te.assert_output( + &["--ignore-file", "custom.ignore", "foo"], + "a.foo + one/b.foo + one/two/c.foo", + ); +} + +/// Ignored files with ripgrep aliases (-u / -uu) +#[test] +fn test_no_ignore_aliases() { + let te = TestEnv::new(DEFAULT_DIRS, DEFAULT_FILES); + + te.assert_output( + &["-u", "foo"], + ".hidden.foo + a.foo + fdignored.foo + gitignored.foo + one/b.foo + one/two/c.foo + one/two/C.Foo2 + one/two/three/d.foo + one/two/three/directory_foo/", + ); +} + +#[cfg(not(windows))] +#[test] +fn test_global_ignore() { + let te = TestEnv::new(DEFAULT_DIRS, DEFAULT_FILES).global_ignore_file("one"); + te.assert_output( + &[], + "a.foo + e1 e2 + symlink", + ); +} + +#[cfg(not(windows))] +#[test_case("--unrestricted", ".hidden.foo +a.foo +fdignored.foo +gitignored.foo +one/b.foo +one/two/c.foo +one/two/C.Foo2 +one/two/three/d.foo +one/two/three/directory_foo/"; "unrestricted")] +#[test_case("--no-ignore", "a.foo +fdignored.foo +gitignored.foo +one/b.foo +one/two/c.foo +one/two/C.Foo2 +one/two/three/d.foo +one/two/three/directory_foo/"; "no-ignore")] +#[test_case("--no-global-ignore-file", "a.foo +one/b.foo +one/two/c.foo +one/two/C.Foo2 +one/two/three/d.foo +one/two/three/directory_foo/"; "no-global-ignore-file")] +fn test_no_global_ignore(flag: &str, expected_output: &str) { + let te = TestEnv::new(DEFAULT_DIRS, DEFAULT_FILES).global_ignore_file("one"); + te.assert_output(&[flag, "foo"], expected_output); +} + +/// Symlinks (--follow) +#[test] +fn test_follow() { + let te = TestEnv::new(DEFAULT_DIRS, DEFAULT_FILES); + + te.assert_output( + &["--follow", "c.foo"], + "one/two/c.foo + one/two/C.Foo2 + symlink/c.foo + symlink/C.Foo2", + ); +} + +// File system boundaries (--one-file-system) +// Limited to Unix because, to the best of my knowledge, there is no easy way to test a use case +// file systems mounted into the tree on Windows. +// Not limiting depth causes massive delay under Darwin, see BurntSushi/ripgrep#1429 +#[test] +#[cfg(unix)] +fn test_file_system_boundaries() { + // Helper function to get the device ID for a given path + // Inspired by https://github.com/BurntSushi/ripgrep/blob/8892bf648cfec111e6e7ddd9f30e932b0371db68/ignore/src/walk.rs#L1693 + fn device_num(path: impl AsRef) -> u64 { + use std::os::unix::fs::MetadataExt; + + path.as_ref().metadata().map(|md| md.dev()).unwrap() + } + + // Can't simulate file system boundaries + let te = TestEnv::new(&[], &[]); + + let dev_null = Path::new("/dev/null"); + + // /dev/null should exist in all sane Unixes. Skip if it doesn't exist for some reason. + // Also skip should it be on the same device as the root partition for some reason. + if !dev_null.is_file() || device_num(dev_null) == device_num("/") { + return; + } + + te.assert_output( + &["--full-path", "--max-depth", "2", "^/dev/null$", "/"], + "/dev/null", + ); + te.assert_output( + &[ + "--one-file-system", + "--full-path", + "--max-depth", + "2", + "^/dev/null$", + "/", + ], + "", + ); +} + +#[test] +fn test_follow_broken_symlink() { + let mut te = TestEnv::new(DEFAULT_DIRS, DEFAULT_FILES); + te.create_broken_symlink("broken_symlink") + .expect("Failed to create broken symlink."); + + te.assert_output( + &["symlink"], + "broken_symlink + symlink", + ); + te.assert_output( + &["--type", "symlink", "symlink"], + "broken_symlink + symlink", + ); + + te.assert_output(&["--type", "file", "symlink"], ""); + + te.assert_output( + &["--follow", "--type", "symlink", "symlink"], + "broken_symlink", + ); + te.assert_output(&["--follow", "--type", "file", "symlink"], ""); +} + +/// Null separator (--print0) +#[test] +fn test_print0() { + let te = TestEnv::new(DEFAULT_DIRS, DEFAULT_FILES); + + te.assert_output( + &["--print0", "foo"], + "./a.fooNULL + ./one/b.fooNULL + ./one/two/C.Foo2NULL + ./one/two/c.fooNULL + ./one/two/three/d.fooNULL + ./one/two/three/directory_foo/NULL", + ); +} + +/// Maximum depth (--max-depth) +#[test] +fn test_max_depth() { + let te = TestEnv::new(DEFAULT_DIRS, DEFAULT_FILES); + + te.assert_output( + &["--max-depth", "3"], + "a.foo + e1 e2 + one/ + one/b.foo + one/two/ + one/two/c.foo + one/two/C.Foo2 + one/two/three/ + symlink", + ); + + te.assert_output( + &["--max-depth", "2"], + "a.foo + e1 e2 + one/ + one/b.foo + one/two/ + symlink", + ); + + te.assert_output( + &["--max-depth", "1"], + "a.foo + e1 e2 + one/ + symlink", + ); +} + +/// Minimum depth (--min-depth) +#[test] +fn test_min_depth() { + let te = TestEnv::new(DEFAULT_DIRS, DEFAULT_FILES); + + te.assert_output( + &["--min-depth", "3"], + "one/two/c.foo + one/two/C.Foo2 + one/two/three/ + one/two/three/d.foo + one/two/three/directory_foo/", + ); + + te.assert_output( + &["--min-depth", "4"], + "one/two/three/d.foo + one/two/three/directory_foo/", + ); +} + +/// Minimum depth with a broken symlink (regression test for #1017) +/// +/// A broken symlink, surfaced while following links, has no depth reported by +/// the walker, so --min-depth used to drop it unconditionally. +#[test] +fn test_min_depth_broken_symlink() { + let mut te = TestEnv::new(DEFAULT_DIRS, DEFAULT_FILES); + te.create_broken_symlink("one/two/broken_symlink") + .expect("Failed to create broken symlink."); + + // The broken symlink sits at depth 3, so it is kept up to that depth. + te.assert_output( + &[ + "--follow", + "--type", + "symlink", + "--min-depth", + "3", + "broken_symlink", + ], + "one/two/broken_symlink", + ); + + // A --min-depth beyond its actual depth must exclude it. + te.assert_output( + &[ + "--follow", + "--type", + "symlink", + "--min-depth", + "4", + "broken_symlink", + ], + "", + ); +} + +/// Minimum depth with a broken symlink combined with --absolute-path (#1017) +/// +/// With --absolute-path the search root is made absolute before walking, so the +/// broken symlink's depth must still be computed relative to that root rather +/// than from the absolute path's full component count. +#[test] +fn test_min_depth_broken_symlink_absolute_path() { + let (mut te, abs_path) = get_test_env_with_abs_path(DEFAULT_DIRS, DEFAULT_FILES); + te.create_broken_symlink("one/two/broken_symlink") + .expect("Failed to create broken symlink."); + + // The broken symlink sits at depth 3 relative to the (absolute) root. + te.assert_output( + &[ + "--follow", + "--absolute-path", + "--type", + "symlink", + "--min-depth", + "3", + "broken_symlink", + ], + &format!("{abs_path}/one/two/broken_symlink"), + ); + + // A --min-depth beyond its actual depth must exclude it. + te.assert_output( + &[ + "--follow", + "--absolute-path", + "--type", + "symlink", + "--min-depth", + "4", + "broken_symlink", + ], + "", + ); +} + +/// Minimum depth with a broken symlink under overlapping search roots (#1017) +/// +/// When two search roots overlap, the walker visits the same broken symlink once +/// per root, at a different depth each time. Here `one/two/broken_symlink` is at +/// depth 2 under root `one` and at depth 1 under root `one/two`, so --min-depth 2 +/// must keep the entry the walker reached via `one` and drop the one it reached +/// via `one/two`. A depth derived from the entry's path cannot distinguish the +/// two visits, since both carry the same path. +#[test] +fn test_min_depth_broken_symlink_overlapping_roots() { + let mut te = TestEnv::new(DEFAULT_DIRS, DEFAULT_FILES); + te.create_broken_symlink("one/two/broken_symlink") + .expect("Failed to create broken symlink."); + + // Only the route through `one` reaches depth 2. + te.assert_output( + &[ + "--follow", + "--type", + "symlink", + "--min-depth", + "2", + "broken_symlink", + "one", + "one/two", + ], + "one/two/broken_symlink", + ); + + // Both routes clear --min-depth 1, so it is reported once per root. + te.assert_output( + &[ + "--follow", + "--type", + "symlink", + "--min-depth", + "1", + "broken_symlink", + "one", + "one/two", + ], + "one/two/broken_symlink + one/two/broken_symlink", + ); +} + +/// Maximum depth with a broken symlink (#1017) +/// +/// A broken symlink must be filtered by --max-depth like any other entry. The +/// default environment also exposes it through the followed `symlink` directory +/// (`symlink -> one/two`), so it is reachable at depth 2 as well as depth 3. +#[test] +fn test_max_depth_broken_symlink() { + let mut te = TestEnv::new(DEFAULT_DIRS, DEFAULT_FILES); + te.create_broken_symlink("one/two/broken_symlink") + .expect("Failed to create broken symlink."); + + // --max-depth 3 keeps both routes to the broken symlink. + te.assert_output( + &[ + "--follow", + "--type", + "symlink", + "--max-depth", + "3", + "broken_symlink", + ], + "one/two/broken_symlink + symlink/broken_symlink", + ); + + // A --max-depth below either route must exclude it. + te.assert_output( + &[ + "--follow", + "--type", + "symlink", + "--max-depth", + "1", + "broken_symlink", + ], + "", + ); +} + +/// Exact depth with a broken symlink (#1017) +/// +/// A broken symlink must be kept only at its exact depth. It is reachable at +/// depth 3 (`one/two/broken_symlink`) and, through the followed `symlink` +/// directory, at depth 2 (`symlink/broken_symlink`). +#[test] +fn test_exact_depth_broken_symlink() { + let mut te = TestEnv::new(DEFAULT_DIRS, DEFAULT_FILES); + te.create_broken_symlink("one/two/broken_symlink") + .expect("Failed to create broken symlink."); + + // Only the depth-3 route matches --exact-depth 3. + te.assert_output( + &[ + "--follow", + "--type", + "symlink", + "--exact-depth", + "3", + "broken_symlink", + ], + "one/two/broken_symlink", + ); + + // Only the depth-2 route (via the followed symlink) matches --exact-depth 2, + // which confirms the depth is computed relative to the search root. + te.assert_output( + &[ + "--follow", + "--type", + "symlink", + "--exact-depth", + "2", + "broken_symlink", + ], + "symlink/broken_symlink", + ); +} + +/// Exact depth (--exact-depth) +#[test] +fn test_exact_depth() { + let te = TestEnv::new(DEFAULT_DIRS, DEFAULT_FILES); + + te.assert_output( + &["--exact-depth", "3"], + "one/two/c.foo + one/two/C.Foo2 + one/two/three/", + ); +} + +/// Pruning (--prune) +#[test] +fn test_prune() { + let dirs = &["foo/bar", "bar/foo", "baz"]; + let files = &[ + "foo/foo.file", + "foo/bar/foo.file", + "bar/foo.file", + "bar/foo/foo.file", + "baz/foo.file", + ]; + + let te = TestEnv::new(dirs, files); + + te.assert_output( + &["foo"], + "foo/ + foo/foo.file + foo/bar/foo.file + bar/foo.file + bar/foo/ + bar/foo/foo.file + baz/foo.file", + ); + + te.assert_output( + &["--prune", "foo"], + "foo/ + bar/foo/ + bar/foo.file + baz/foo.file", + ); +} + +/// Absolute paths (--absolute-path) +#[test] +fn test_absolute_path() { + let (te, abs_path) = get_test_env_with_abs_path(DEFAULT_DIRS, DEFAULT_FILES); + + te.assert_output( + &["--absolute-path"], + &format!( + "{abs_path}/a.foo + {abs_path}/e1 e2 + {abs_path}/one/ + {abs_path}/one/b.foo + {abs_path}/one/two/ + {abs_path}/one/two/c.foo + {abs_path}/one/two/C.Foo2 + {abs_path}/one/two/three/ + {abs_path}/one/two/three/d.foo + {abs_path}/one/two/three/directory_foo/ + {abs_path}/symlink", + abs_path = abs_path + ), + ); + + te.assert_output( + &["--absolute-path", "foo"], + &format!( + "{abs_path}/a.foo + {abs_path}/one/b.foo + {abs_path}/one/two/c.foo + {abs_path}/one/two/C.Foo2 + {abs_path}/one/two/three/d.foo + {abs_path}/one/two/three/directory_foo/", + abs_path = abs_path + ), + ); +} + +/// Show absolute paths if the path argument is absolute +#[test] +fn test_implicit_absolute_path() { + let (te, abs_path) = get_test_env_with_abs_path(DEFAULT_DIRS, DEFAULT_FILES); + + te.assert_output( + &["foo", &abs_path], + &format!( + "{abs_path}/a.foo + {abs_path}/one/b.foo + {abs_path}/one/two/c.foo + {abs_path}/one/two/C.Foo2 + {abs_path}/one/two/three/d.foo + {abs_path}/one/two/three/directory_foo/", + abs_path = abs_path + ), + ); +} + +/// Absolute paths should be normalized +#[test] +fn test_normalized_absolute_path() { + let (te, abs_path) = get_test_env_with_abs_path(DEFAULT_DIRS, DEFAULT_FILES); + + te.assert_output_subdirectory( + "one", + &["--absolute-path", "foo", ".."], + &format!( + "{abs_path}/a.foo + {abs_path}/one/b.foo + {abs_path}/one/two/c.foo + {abs_path}/one/two/C.Foo2 + {abs_path}/one/two/three/d.foo + {abs_path}/one/two/three/directory_foo/", + abs_path = abs_path + ), + ); +} + +/// File type filter (--type) +#[test] +fn test_type() { + let te = TestEnv::new(DEFAULT_DIRS, DEFAULT_FILES); + + te.assert_output( + &["--type", "f"], + "a.foo + e1 e2 + one/b.foo + one/two/c.foo + one/two/C.Foo2 + one/two/three/d.foo", + ); + + te.assert_output(&["--type", "f", "e1"], "e1 e2"); + + te.assert_output( + &["--type", "d"], + "one/ + one/two/ + one/two/three/ + one/two/three/directory_foo/", + ); + + te.assert_output( + &["--type", "d", "--type", "l"], + "one/ + one/two/ + one/two/three/ + one/two/three/directory_foo/ + symlink", + ); + + te.assert_output(&["--type", "l"], "symlink"); +} + +/// Test `--type executable` +#[cfg(unix)] +#[test] +fn test_type_executable() { + use std::os::unix::fs::OpenOptionsExt; + + // This test assumes the current user isn't root + // (otherwise if the executable bit is set for any level, it is executable for the current + // user) + if Uid::current().is_root() { + return; + } + + let te = TestEnv::new(DEFAULT_DIRS, DEFAULT_FILES); + + fs::OpenOptions::new() + .create_new(true) + .truncate(true) + .write(true) + .mode(0o777) + .open(te.test_root().join("executable-file.sh")) + .unwrap(); + + fs::OpenOptions::new() + .create(true) + .truncate(true) + .write(true) + .mode(0o645) + .open(te.test_root().join("not-user-executable-file.sh")) + .unwrap(); + + te.assert_output(&["--type", "executable"], "executable-file.sh"); + + te.assert_output( + &["--type", "executable", "--type", "directory"], + "executable-file.sh + one/ + one/two/ + one/two/three/ + one/two/three/directory_foo/", + ); +} + +/// Test `--type empty` +#[test] +fn test_type_empty() { + let te = TestEnv::new(&["dir_empty", "dir_nonempty"], &[]); + + create_file_with_size(te.test_root().join("0_bytes.foo"), 0); + create_file_with_size(te.test_root().join("5_bytes.foo"), 5); + + create_file_with_size(te.test_root().join("dir_nonempty").join("2_bytes.foo"), 2); + + te.assert_output( + &["--type", "empty"], + "0_bytes.foo + dir_empty/", + ); + + te.assert_output( + &["--type", "empty", "--type", "file", "--type", "directory"], + "0_bytes.foo + dir_empty/", + ); + + te.assert_output(&["--type", "empty", "--type", "file"], "0_bytes.foo"); + + te.assert_output(&["--type", "empty", "--type", "directory"], "dir_empty/"); +} + +/// File extension (--extension) +#[test] +fn test_extension() { + let te = TestEnv::new(DEFAULT_DIRS, DEFAULT_FILES); + + te.assert_output( + &["--extension", "foo"], + "a.foo + one/b.foo + one/two/c.foo + one/two/three/d.foo", + ); + + te.assert_output( + &["--extension", ".foo"], + "a.foo + one/b.foo + one/two/c.foo + one/two/three/d.foo", + ); + + te.assert_output( + &["--extension", ".foo", "--extension", "foo2"], + "a.foo + one/b.foo + one/two/c.foo + one/two/three/d.foo + one/two/C.Foo2", + ); + + te.assert_output(&["--extension", ".foo", "a"], "a.foo"); + + te.assert_output(&["--extension", "foo2"], "one/two/C.Foo2"); + + let te2 = TestEnv::new(&[], &["spam.bar.baz", "egg.bar.baz", "yolk.bar.baz.sig"]); + + te2.assert_output( + &["--extension", ".bar.baz"], + "spam.bar.baz + egg.bar.baz", + ); + + te2.assert_output(&["--extension", "sig"], "yolk.bar.baz.sig"); + + te2.assert_output(&["--extension", "bar.baz.sig"], "yolk.bar.baz.sig"); + + let te3 = TestEnv::new(&[], &["latin1.e\u{301}xt", "smiley.☻"]); + + te3.assert_output(&["--extension", "☻"], "smiley.☻"); + + te3.assert_output(&["--extension", ".e\u{301}xt"], "latin1.e\u{301}xt"); + + let te4 = TestEnv::new(&[], &[".hidden", "test.hidden"]); + + te4.assert_output(&["--hidden", "--extension", ".hidden"], "test.hidden"); +} + +/// No file extension (test for the pattern provided in the --help text) +#[test] +fn test_no_extension() { + let te = TestEnv::new( + DEFAULT_DIRS, + &["a.foo", "aa", "one/b.foo", "one/bb", "one/two/three/d"], + ); + + te.assert_output( + &["^[^.]+$"], + "aa + one/ + one/bb + one/two/ + one/two/three/ + one/two/three/d + one/two/three/directory_foo/ + symlink", + ); + + te.assert_output( + &["^[^.]+$", "--type", "file"], + "aa + one/bb + one/two/three/d", + ); +} + +/// Symlink as search directory +#[test] +fn test_symlink_as_root() { + let mut te = TestEnv::new(DEFAULT_DIRS, DEFAULT_FILES); + te.create_broken_symlink("broken_symlink") + .expect("Failed to create broken symlink."); + + // From: http://pubs.opengroup.org/onlinepubs/9699919799/functions/getcwd.html + // The getcwd() function shall place an absolute pathname of the current working directory in + // the array pointed to by buf, and return buf. The pathname shall contain no components that + // are dot or dot-dot, or are symbolic links. + // + // Key points: + // 1. The path of the current working directory of a Unix process cannot contain symlinks. + // 2. The path of the current working directory of a Windows process can contain symlinks. + // + // More: + // 1. On Windows, symlinks are resolved after the ".." component. + // 2. On Unix, symlinks are resolved immediately as encountered. + + let parent_parent = if cfg!(windows) { ".." } else { "../.." }; + te.assert_output_subdirectory( + "symlink", + &["", parent_parent], + &format!( + "{dir}/a.foo + {dir}/broken_symlink + {dir}/e1 e2 + {dir}/one/ + {dir}/one/b.foo + {dir}/one/two/ + {dir}/one/two/c.foo + {dir}/one/two/C.Foo2 + {dir}/one/two/three/ + {dir}/one/two/three/d.foo + {dir}/one/two/three/directory_foo/ + {dir}/symlink", + dir = parent_parent + ), + ); +} + +#[test] +fn test_symlink_and_absolute_path() { + let (te, abs_path) = get_test_env_with_abs_path(DEFAULT_DIRS, DEFAULT_FILES); + + let expected_path = if cfg!(windows) { "symlink" } else { "one/two" }; + + te.assert_output_subdirectory( + "symlink", + &["--absolute-path"], + &format!( + "{abs_path}/{expected_path}/c.foo + {abs_path}/{expected_path}/C.Foo2 + {abs_path}/{expected_path}/three/ + {abs_path}/{expected_path}/three/d.foo + {abs_path}/{expected_path}/three/directory_foo/", + abs_path = abs_path, + expected_path = expected_path + ), + ); +} + +#[test] +fn test_symlink_as_absolute_root() { + let (te, abs_path) = get_test_env_with_abs_path(DEFAULT_DIRS, DEFAULT_FILES); + + te.assert_output( + &["", &format!("{abs_path}/symlink")], + &format!( + "{abs_path}/symlink/c.foo + {abs_path}/symlink/C.Foo2 + {abs_path}/symlink/three/ + {abs_path}/symlink/three/d.foo + {abs_path}/symlink/three/directory_foo/", + abs_path = abs_path + ), + ); +} + +#[test] +fn test_symlink_and_full_path() { + let (te, abs_path) = get_test_env_with_abs_path(DEFAULT_DIRS, DEFAULT_FILES); + let root = te.system_root(); + let prefix = escape(&root.to_string_lossy()); + + let expected_path = if cfg!(windows) { "symlink" } else { "one/two" }; + + te.assert_output_subdirectory( + "symlink", + &[ + "--absolute-path", + "--full-path", + &format!("^{prefix}.*three"), + ], + &format!( + "{abs_path}/{expected_path}/three/ + {abs_path}/{expected_path}/three/d.foo + {abs_path}/{expected_path}/three/directory_foo/", + abs_path = abs_path, + expected_path = expected_path + ), + ); +} + +#[test] +fn test_symlink_and_full_path_abs_path() { + let (te, abs_path) = get_test_env_with_abs_path(DEFAULT_DIRS, DEFAULT_FILES); + let root = te.system_root(); + let prefix = escape(&root.to_string_lossy()); + te.assert_output( + &[ + "--full-path", + &format!("^{prefix}.*symlink.*three"), + &format!("{abs_path}/symlink"), + ], + &format!( + "{abs_path}/symlink/three/ + {abs_path}/symlink/three/d.foo + {abs_path}/symlink/three/directory_foo/", + abs_path = abs_path + ), + ); +} +/// Exclude patterns (--exclude) +#[test] +fn test_excludes() { + let te = TestEnv::new(DEFAULT_DIRS, DEFAULT_FILES); + + te.assert_output( + &["--exclude", "*.foo"], + "one/ + one/two/ + one/two/C.Foo2 + one/two/three/ + one/two/three/directory_foo/ + e1 e2 + symlink", + ); + + te.assert_output( + &["--exclude", "*.foo", "--exclude", "*.Foo2"], + "one/ + one/two/ + one/two/three/ + one/two/three/directory_foo/ + e1 e2 + symlink", + ); + + te.assert_output( + &["--exclude", "*.foo", "--exclude", "*.Foo2", "foo"], + "one/two/three/directory_foo/", + ); + + te.assert_output( + &["--exclude", "one/two/", "foo"], + "a.foo + one/b.foo", + ); + + te.assert_output( + &["--exclude", "one/**/*.foo"], + "a.foo + e1 e2 + one/ + one/two/ + one/two/C.Foo2 + one/two/three/ + one/two/three/directory_foo/ + symlink", + ); +} + +#[test] +fn format() { + let te = TestEnv::new(DEFAULT_DIRS, DEFAULT_FILES); + + te.assert_output( + &["--format", "path={}", "--path-separator=/"], + "path=a.foo + path=e1 e2 + path=one + path=one/b.foo + path=one/two + path=one/two/C.Foo2 + path=one/two/c.foo + path=one/two/three + path=one/two/three/d.foo + path=one/two/three/directory_foo + path=symlink", + ); + + te.assert_output( + &["foo", "--format", "noExt={.}", "--path-separator=/"], + "noExt=a + noExt=one/b + noExt=one/two/C + noExt=one/two/c + noExt=one/two/three/d + noExt=one/two/three/directory_foo", + ); + + te.assert_output( + &["foo", "--format", "basename={/}", "--path-separator=/"], + "basename=a.foo + basename=b.foo + basename=C.Foo2 + basename=c.foo + basename=d.foo + basename=directory_foo", + ); + + te.assert_output( + &["foo", "--format", "name={/.}", "--path-separator=/"], + "name=a + name=b + name=C + name=c + name=d + name=directory_foo", + ); + + te.assert_output( + &["foo", "--format", "parent={//}", "--path-separator=/"], + "parent=. + parent=one + parent=one/two + parent=one/two + parent=one/two/three + parent=one/two/three", + ); + + // Templates may start with '-' (e.g. markdown list items); see #2126. + te.assert_output( + &["foo", "--format", "- {/.}", "--path-separator=/"], + "- a + - b + - C + - c + - d + - directory_foo", + ); +} + +/// Shell script execution (--exec) +#[test] +fn test_exec() { + let (te, abs_path) = get_test_env_with_abs_path(DEFAULT_DIRS, DEFAULT_FILES); + // TODO Windows tests: D:file.txt \file.txt \\server\share\file.txt ... + if !cfg!(windows) { + te.assert_output( + &["--absolute-path", "foo", "--exec", "echo"], + &format!( + "{abs_path}/a.foo + {abs_path}/one/b.foo + {abs_path}/one/two/C.Foo2 + {abs_path}/one/two/c.foo + {abs_path}/one/two/three/d.foo + {abs_path}/one/two/three/directory_foo", + abs_path = abs_path + ), + ); + + te.assert_output( + &["foo", "--exec", "echo", "{}"], + "./a.foo + ./one/b.foo + ./one/two/C.Foo2 + ./one/two/c.foo + ./one/two/three/d.foo + ./one/two/three/directory_foo", + ); + + te.assert_output( + &["foo", "--strip-cwd-prefix", "--exec", "echo", "{}"], + "a.foo + one/b.foo + one/two/C.Foo2 + one/two/c.foo + one/two/three/d.foo + one/two/three/directory_foo", + ); + + te.assert_output( + &["foo", "--exec", "echo", "{.}"], + "a + one/b + one/two/C + one/two/c + one/two/three/d + one/two/three/directory_foo", + ); + + te.assert_output( + &["foo", "--exec", "echo", "{/}"], + "a.foo + b.foo + C.Foo2 + c.foo + d.foo + directory_foo", + ); + + te.assert_output( + &["foo", "--exec", "echo", "{/.}"], + "a + b + C + c + d + directory_foo", + ); + + te.assert_output( + &["foo", "--exec", "echo", "{//}"], + ". + ./one + ./one/two + ./one/two + ./one/two/three + ./one/two/three", + ); + + te.assert_output(&["e1", "--exec", "printf", "%s.%s\n"], "./e1 e2."); + } +} + +// TODO test for windows +#[cfg(not(windows))] +#[test] +fn test_exec_multi() { + let (te, abs_path) = get_test_env_with_abs_path(DEFAULT_DIRS, DEFAULT_FILES); + + te.assert_output( + &[ + "--absolute-path", + "foo", + "--exec", + "echo", + ";", + "--exec", + "echo", + "test", + "{/}", + ], + &format!( + "{abs_path}/a.foo + {abs_path}/one/b.foo + {abs_path}/one/two/C.Foo2 + {abs_path}/one/two/c.foo + {abs_path}/one/two/three/d.foo + {abs_path}/one/two/three/directory_foo + test a.foo + test b.foo + test C.Foo2 + test c.foo + test d.foo + test directory_foo", + abs_path = abs_path + ), + ); + + te.assert_output( + &[ + "e1", "--exec", "echo", "{.}", ";", "--exec", "echo", "{/}", ";", "--exec", "echo", + "{//}", ";", "--exec", "echo", "{/.}", + ], + "e1 e2 + e1 e2 + . + e1 e2", + ); + + // We use printf here because we need to suppress a newline and + // echo -n is not POSIX-compliant. + te.assert_output( + &[ + "foo", "--exec", "printf", "%s", "{/}: ", ";", "--exec", "printf", "%s\\n", "{//}", + ], + "a.foo: . + b.foo: ./one + C.Foo2: ./one/two + c.foo: ./one/two + d.foo: ./one/two/three + directory_foo: ./one/two/three", + ); +} + +#[cfg(not(windows))] +#[test] +fn test_exec_nulls() { + let te = TestEnv::new(DEFAULT_DIRS, DEFAULT_FILES); + te.assert_output( + &["foo", "--print0", "--exec", "printf", "p=%s"], + "p=./a.fooNULL + p=./one/b.fooNULL + p=./one/two/C.Foo2NULL + p=./one/two/c.fooNULL + p=./one/two/three/d.fooNULL + p=./one/two/three/directory_fooNULL", + ); +} + +#[test] +fn test_exec_batch() { + let (te, abs_path) = get_test_env_with_abs_path(DEFAULT_DIRS, DEFAULT_FILES); + let te = te.normalize_line(true); + + // TODO Test for windows + if !cfg!(windows) { + te.assert_output( + &["--absolute-path", "foo", "--exec-batch", "echo"], + &format!( + "{abs_path}/a.foo {abs_path}/one/b.foo {abs_path}/one/two/C.Foo2 {abs_path}/one/two/c.foo {abs_path}/one/two/three/d.foo {abs_path}/one/two/three/directory_foo", + abs_path = abs_path + ), + ); + + te.assert_output( + &["foo", "--exec-batch", "echo", "{}"], + "./a.foo ./one/b.foo ./one/two/C.Foo2 ./one/two/c.foo ./one/two/three/d.foo ./one/two/three/directory_foo", + ); + + te.assert_output( + &["foo", "--strip-cwd-prefix", "--exec-batch", "echo", "{}"], + "a.foo one/b.foo one/two/C.Foo2 one/two/c.foo one/two/three/d.foo one/two/three/directory_foo", + ); + + te.assert_output( + &["foo", "--exec-batch", "echo", "{/}"], + "a.foo b.foo C.Foo2 c.foo d.foo directory_foo", + ); + + te.assert_output( + &["no_match", "--exec-batch", "echo", "Matched: ", "{/}"], + "", + ); + + te.assert_failure_with_error( + &["foo", "--exec-batch", "echo", "{}", "{}"], + "error: Only one placeholder allowed for batch commands\n\ + \n\ + Usage: fd [OPTIONS] [pattern] [path]...\n\ + \n\ + For more information, try '--help'.\n\ + ", + ); + + te.assert_failure_with_error( + &["foo", "--exec-batch", "echo", "{/}", ";", "-x", "echo"], + "error: the argument '--exec-batch ...' cannot be used with '--exec ...'\n\ + \n\ + Usage: fd --exec-batch ... [path]...\n\ + \n\ + For more information, try '--help'.\n\ + ", + ); + + te.assert_failure_with_error( + &["foo", "--exec-batch"], + "error: a value is required for '--exec-batch ...' but none was supplied\n\ + \n\ + For more information, try '--help'.\n\ + ", + ); + + te.assert_failure_with_error( + &["foo", "--exec-batch", "echo {}"], + "error: First argument of --exec-batch must be a fixed executable, not a placeholder\n\ + \n\ + Usage: fd [OPTIONS] [pattern] [path]...\n\ + \n\ + For more information, try '--help'.\n\ + ", + ); + + te.assert_failure_with_error(&["a.foo", "--exec-batch", "bash", "-c", "exit 1"], ""); + } +} + +#[test] +fn test_exec_batch_multi() { + // TODO test for windows + if cfg!(windows) { + return; + } + let te = TestEnv::new(DEFAULT_DIRS, DEFAULT_FILES); + + let output = te.assert_success_and_get_output( + ".", + &[ + "foo", + "--exec-batch", + "echo", + "{}", + ";", + "--exec-batch", + "echo", + "{/}", + ], + ); + let stdout = std::str::from_utf8(&output.stdout).unwrap(); + let lines: Vec<_> = stdout + .lines() + .map(|l| { + let mut words: Vec<_> = l.split_whitespace().collect(); + words.sort_unstable(); + words + }) + .collect(); + + assert_eq!( + lines, + &[ + [ + "./a.foo", + "./one/b.foo", + "./one/two/C.Foo2", + "./one/two/c.foo", + "./one/two/three/d.foo", + "./one/two/three/directory_foo" + ], + [ + "C.Foo2", + "a.foo", + "b.foo", + "c.foo", + "d.foo", + "directory_foo" + ], + ] + ); + + te.assert_failure_with_error( + &[ + "a.foo", + "--exec-batch", + "echo", + ";", + "--exec-batch", + "bash", + "-c", + "exit 1", + ], + "", + ); +} + +#[test] +fn test_exec_batch_with_limit() { + // TODO Test for windows + if cfg!(windows) { + return; + } + + let te = TestEnv::new(DEFAULT_DIRS, DEFAULT_FILES); + + let output = te.assert_success_and_get_output( + ".", + &["foo", "--batch-size=2", "--exec-batch", "echo", "{}"], + ); + let stdout = String::from_utf8_lossy(&output.stdout); + + for line in stdout.lines() { + assert_eq!(2, line.split_whitespace().count()); + } + + let mut paths: Vec<_> = stdout + .lines() + .flat_map(|line| line.split_whitespace()) + .collect(); + paths.sort_unstable(); + assert_eq!( + &paths, + &[ + "./a.foo", + "./one/b.foo", + "./one/two/C.Foo2", + "./one/two/c.foo", + "./one/two/three/d.foo", + "./one/two/three/directory_foo" + ], + ); +} + +/// Shell script execution (--exec) with a custom --path-separator +#[test] +fn test_exec_with_separator() { + // On Windows, `echo` is only a shell builtin unless an `echo` executable is installed. + if cfg!(windows) && std::process::Command::new("echo").output().is_err() { + return; + } + + let (te, abs_path) = get_test_env_with_abs_path(DEFAULT_DIRS, DEFAULT_FILES); + te.assert_output( + &[ + "--path-separator=#", + "--absolute-path", + "foo", + "--exec", + "echo", + ], + &format!( + "{abs_path}#a.foo + {abs_path}#one#b.foo + {abs_path}#one#two#C.Foo2 + {abs_path}#one#two#c.foo + {abs_path}#one#two#three#d.foo + {abs_path}#one#two#three#directory_foo", + abs_path = abs_path.replace(std::path::MAIN_SEPARATOR, "#"), + ), + ); + + te.assert_output( + &["--path-separator=#", "foo", "--exec", "echo", "{}"], + ".#a.foo + .#one#b.foo + .#one#two#C.Foo2 + .#one#two#c.foo + .#one#two#three#d.foo + .#one#two#three#directory_foo", + ); + + te.assert_output( + &["--path-separator=#", "foo", "--exec", "echo", "{.}"], + "a + one#b + one#two#C + one#two#c + one#two#three#d + one#two#three#directory_foo", + ); + + te.assert_output( + &["--path-separator=#", "foo", "--exec", "echo", "{/}"], + "a.foo + b.foo + C.Foo2 + c.foo + d.foo + directory_foo", + ); + + te.assert_output( + &["--path-separator=#", "foo", "--exec", "echo", "{/.}"], + "a + b + C + c + d + directory_foo", + ); + + te.assert_output( + &["--path-separator=#", "foo", "--exec", "echo", "{//}"], + ". + .#one + .#one#two + .#one#two + .#one#two#three + .#one#two#three", + ); + + te.assert_output( + &["--path-separator=#", "e1", "--exec", "printf", "%s.%s\n"], + ".#e1 e2.", + ); +} + +/// Non-zero exit code (--quiet) +#[test] +fn test_quiet() { + let dirs = &[]; + let files = &["a.foo", "b.foo"]; + let te = TestEnv::new(dirs, files); + + te.assert_output(&["-q"], ""); + te.assert_output(&["--quiet"], ""); + te.assert_output(&["--has-results"], ""); + te.assert_failure_with_error(&["--quiet", "c.foo"], "") +} + +/// Literal search (--fixed-strings) +#[test] +fn test_fixed_strings() { + let dirs = &["test1", "test2"]; + let files = &["test1/a.foo", "test1/a_foo", "test2/Download (1).tar.gz"]; + let te = TestEnv::new(dirs, files); + + // Regex search, dot is treated as "any character" + te.assert_output( + &["a.foo"], + "test1/a.foo + test1/a_foo", + ); + + // Literal search, dot is treated as character + te.assert_output(&["--fixed-strings", "a.foo"], "test1/a.foo"); + + // Regex search, parens are treated as group + te.assert_output(&["download (1)"], ""); + + // Literal search, parens are treated as characters. Case-insensitive by default. + te.assert_output( + &["--fixed-strings", "download (1)"], + "test2/Download (1).tar.gz", + ); + + // Combine with --case-sensitive (unmatched). + te.assert_output(&["--fixed-strings", "--case-sensitive", "download (1)"], ""); + + // Combine with --case-sensitive (matched). + te.assert_output( + &["--fixed-strings", "--case-sensitive", "Download (1)"], + "test2/Download (1).tar.gz", + ); +} + +/// Literal search, non-substring (--exact) +#[test] +fn test_exact_literal_nonsubstring() { + let dirs = &["test1", "test2"]; + let files = &[ + "test1/a.foo", + "test1/aa.foo", + "test1/a.food", + "test1/ca.food", + "test1/a_foo", + "test2/Download (1).tar.gz", + ]; + let te = TestEnv::new(dirs, files); + + // Literal search, dot is treated as character. Should match only the exact name, + // not "aa.foo", "a.food", or "ca.food". + te.assert_output(&["--exact", "a.foo"], "test1/a.foo"); + + // Literal search, parens are treated as characters. Substring should not match. + te.assert_output(&["--exact", "download (1)"], ""); + + // Literal search, parens are treated as characters. Case-insensitive by default. + te.assert_output( + &["--exact", "download (1).tar.gz"], + "test2/Download (1).tar.gz", + ); + + // Combine with --case-sensitive, should not match due to case mismatch. + te.assert_output(&["--exact", "--case-sensitive", "download (1).tar.gz"], ""); + + // Combine with --case-sensitive, exact match should match. + te.assert_output( + &["--exact", "--case-sensitive", "Download (1).tar.gz"], + "test2/Download (1).tar.gz", + ); +} + +/// Filenames with invalid UTF-8 sequences +#[cfg(target_os = "linux")] +#[test] +fn test_invalid_utf8() { + use std::ffi::OsStr; + use std::os::unix::ffi::OsStrExt; + + let dirs = &["test1"]; + let files = &[]; + let te = TestEnv::new(dirs, files); + + fs::File::create( + te.test_root() + .join(OsStr::from_bytes(b"test1/test_\xFEinvalid.txt")), + ) + .unwrap(); + + te.assert_output(&["", "test1/"], "test1/test_�invalid.txt"); + + te.assert_output(&["invalid", "test1/"], "test1/test_�invalid.txt"); + + // Should not be found under a different extension + te.assert_output(&["-e", "zip", "", "test1/"], ""); +} + +/// Filtering for file size (--size) +#[test] +fn test_size() { + let te = TestEnv::new(&[], &[]); + + create_file_with_size(te.test_root().join("0_bytes.foo"), 0); + create_file_with_size(te.test_root().join("11_bytes.foo"), 11); + create_file_with_size(te.test_root().join("30_bytes.foo"), 30); + create_file_with_size(te.test_root().join("3_kilobytes.foo"), 3 * 1000); + create_file_with_size(te.test_root().join("4_kibibytes.foo"), 4 * 1024); + + // Zero and non-zero sized files. + te.assert_output( + &["", "--size", "+0B"], + "0_bytes.foo + 11_bytes.foo + 30_bytes.foo + 3_kilobytes.foo + 4_kibibytes.foo", + ); + + // Zero sized files. + te.assert_output(&["", "--size", "-0B"], "0_bytes.foo"); + te.assert_output(&["", "--size", "0B"], "0_bytes.foo"); + te.assert_output(&["", "--size=0B"], "0_bytes.foo"); + te.assert_output(&["", "-S", "0B"], "0_bytes.foo"); + + // Files with 2 bytes or more. + te.assert_output( + &["", "--size", "+2B"], + "11_bytes.foo + 30_bytes.foo + 3_kilobytes.foo + 4_kibibytes.foo", + ); + + // Files with 2 bytes or less. + te.assert_output(&["", "--size", "-2B"], "0_bytes.foo"); + + // Files with size between 1 byte and 11 bytes. + te.assert_output(&["", "--size", "+1B", "--size", "-11B"], "11_bytes.foo"); + + // Files with size equal 11 bytes. + te.assert_output(&["", "--size", "11B"], "11_bytes.foo"); + + // Files with size between 1 byte and 30 bytes. + te.assert_output( + &["", "--size", "+1B", "--size", "-30B"], + "11_bytes.foo + 30_bytes.foo", + ); + + // Combine with a search pattern + te.assert_output(&["^11_", "--size", "+1B", "--size", "-30B"], "11_bytes.foo"); + + // Files with size between 12 and 30 bytes. + te.assert_output(&["", "--size", "+12B", "--size", "-30B"], "30_bytes.foo"); + + // Files with size between 31 and 100 bytes. + te.assert_output(&["", "--size", "+31B", "--size", "-100B"], ""); + + // Files with size between 3 kibibytes and 5 kibibytes. + te.assert_output(&["", "--size", "+3ki", "--size", "-5ki"], "4_kibibytes.foo"); + + // Files with size between 3 kilobytes and 5 kilobytes. + te.assert_output( + &["", "--size", "+3k", "--size", "-5k"], + "3_kilobytes.foo + 4_kibibytes.foo", + ); + + // Files with size greater than 3 kilobytes and less than 3 kibibytes. + te.assert_output(&["", "--size", "+3k", "--size", "-3ki"], "3_kilobytes.foo"); + + // Files with size equal 4 kibibytes. + te.assert_output(&["", "--size", "+4ki", "--size", "-4ki"], "4_kibibytes.foo"); + te.assert_output(&["", "--size", "4ki"], "4_kibibytes.foo"); +} + +#[cfg(test)] +fn create_file_with_modified>(path: P, duration_in_secs: u64) { + let st = SystemTime::now() - Duration::from_secs(duration_in_secs); + let ft = filetime::FileTime::from_system_time(st); + fs::File::create(&path).expect("creation failed"); + filetime::set_file_times(&path, ft, ft).expect("time modification failed"); +} + +#[cfg(test)] +fn remove_symlink>(path: P) { + #[cfg(unix)] + fs::remove_file(path).expect("remove symlink"); + + // On Windows, symlinks remember whether they point to files or directories, so try both + #[cfg(windows)] + fs::remove_file(path.as_ref()) + .or_else(|_| fs::remove_dir(path.as_ref())) + .expect("remove symlink"); +} + +#[test] +fn test_modified_relative() { + let te = TestEnv::new(&[], &[]); + remove_symlink(te.test_root().join("symlink")); + create_file_with_modified(te.test_root().join("foo_0_now"), 0); + create_file_with_modified(te.test_root().join("bar_1_min"), 60); + create_file_with_modified(te.test_root().join("foo_10_min"), 600); + create_file_with_modified(te.test_root().join("bar_1_h"), 60 * 60); + create_file_with_modified(te.test_root().join("foo_2_h"), 2 * 60 * 60); + create_file_with_modified(te.test_root().join("bar_1_day"), 24 * 60 * 60); + + te.assert_output( + &["", "--changed-within", "15min"], + "foo_0_now + bar_1_min + foo_10_min", + ); + + te.assert_output( + &["", "--change-older-than", "15min"], + "bar_1_h + foo_2_h + bar_1_day", + ); + + te.assert_output( + &["foo", "--changed-within", "12h"], + "foo_0_now + foo_10_min + foo_2_h", + ); +} + +#[cfg(test)] +fn change_file_modified>(path: P, iso_date: &str) { + let st = iso_date + .parse::() + .map(SystemTime::from) + .expect("invalid date"); + let ft = filetime::FileTime::from_system_time(st); + filetime::set_file_times(path, ft, ft).expect("time modification failde"); +} + +#[test] +fn test_modified_absolute() { + let te = TestEnv::new(&[], &["15mar2018", "30dec2017"]); + remove_symlink(te.test_root().join("symlink")); + change_file_modified(te.test_root().join("15mar2018"), "2018-03-15T12:00:00Z"); + change_file_modified(te.test_root().join("30dec2017"), "2017-12-30T23:59:00Z"); + + te.assert_output( + &["", "--change-newer-than", "2018-01-01 00:00:00"], + "15mar2018", + ); + te.assert_output( + &["", "--changed-before", "2018-01-01 00:00:00"], + "30dec2017", + ); +} + +#[cfg(unix)] +#[test] +fn test_owner_ignore_all() { + let te = TestEnv::new(DEFAULT_DIRS, DEFAULT_FILES); + te.assert_output(&["--owner", ":", "a.foo"], "a.foo"); + te.assert_output(&["--owner", "", "a.foo"], "a.foo"); +} + +#[cfg(unix)] +#[test] +fn test_owner_current_user() { + let te = TestEnv::new(DEFAULT_DIRS, DEFAULT_FILES); + let uid = Uid::current(); + te.assert_output(&["--owner", &uid.to_string(), "a.foo"], "a.foo"); + if let Ok(Some(user)) = User::from_uid(uid) { + te.assert_output(&["--owner", &user.name, "a.foo"], "a.foo"); + } +} + +#[cfg(unix)] +#[test] +fn test_owner_current_group() { + let te = TestEnv::new(DEFAULT_DIRS, DEFAULT_FILES); + let gid = Gid::current(); + te.assert_output(&["--owner", &format!(":{gid}"), "a.foo"], "a.foo"); + if let Ok(Some(group)) = Group::from_gid(gid) { + te.assert_output(&["--owner", &format!(":{}", group.name), "a.foo"], "a.foo"); + } +} + +#[cfg(target_os = "linux")] +#[test] +fn test_owner_root() { + // This test assumes the current user isn't root + if Uid::current().is_root() || Gid::current() == Gid::from_raw(0) { + return; + } + let te = TestEnv::new(DEFAULT_DIRS, DEFAULT_FILES); + te.assert_output(&["--owner", "root", "a.foo"], ""); + te.assert_output(&["--owner", "0", "a.foo"], ""); + te.assert_output(&["--owner", ":root", "a.foo"], ""); + te.assert_output(&["--owner", ":0", "a.foo"], ""); +} + +#[test] +fn test_custom_path_separator() { + let te = TestEnv::new(DEFAULT_DIRS, DEFAULT_FILES); + + te.assert_output( + &["foo", "one", "--path-separator", "="], + "one=b.foo + one=two=c.foo + one=two=C.Foo2 + one=two=three=d.foo + one=two=three=directory_foo=", + ); +} + +#[test] +fn test_base_directory() { + let te = TestEnv::new(DEFAULT_DIRS, DEFAULT_FILES); + + te.assert_output( + &["--base-directory", "one"], + "b.foo + two/ + two/c.foo + two/C.Foo2 + two/three/ + two/three/d.foo + two/three/directory_foo/", + ); + + te.assert_output( + &["--base-directory", "one/two/", "foo"], + "c.foo + C.Foo2 + three/d.foo + three/directory_foo/", + ); + + // Explicit root path + te.assert_output( + &["--base-directory", "one", "foo", "two"], + "two/c.foo + two/C.Foo2 + two/three/d.foo + two/three/directory_foo/", + ); + + // Ignore base directory when absolute path is used + let (te, abs_path) = get_test_env_with_abs_path(DEFAULT_DIRS, DEFAULT_FILES); + let abs_base_dir = &format!("{abs_path}/one/two/", abs_path = abs_path); + te.assert_output( + &["--base-directory", abs_base_dir, "foo", &abs_path], + &format!( + "{abs_path}/a.foo + {abs_path}/one/b.foo + {abs_path}/one/two/c.foo + {abs_path}/one/two/C.Foo2 + {abs_path}/one/two/three/d.foo + {abs_path}/one/two/three/directory_foo/", + abs_path = abs_path + ), + ); +} + +#[test] +fn test_max_results() { + let te = TestEnv::new(DEFAULT_DIRS, DEFAULT_FILES); + + // Unrestricted + te.assert_output( + &["--max-results=0", "c.foo"], + "one/two/C.Foo2 + one/two/c.foo", + ); + + // Limited to two results + te.assert_output( + &["--max-results=2", "c.foo"], + "one/two/C.Foo2 + one/two/c.foo", + ); + + // Limited to one result. We could find either C.Foo2 or c.foo + let assert_just_one_result_with_option = |option| { + let output = te.assert_success_and_get_output(".", &[option, "c.foo"]); + let stdout = String::from_utf8_lossy(&output.stdout) + .trim() + .replace(&std::path::MAIN_SEPARATOR.to_string(), "/"); + assert!(stdout == "one/two/C.Foo2" || stdout == "one/two/c.foo"); + }; + assert_just_one_result_with_option("--max-results=1"); + assert_just_one_result_with_option("-1"); + + // check that --max-results & -1 conflict with --exec + te.assert_failure(&["thing", "--max-results=0", "--exec=cat"]); + te.assert_failure(&["thing", "-1", "--exec=cat"]); + te.assert_failure(&["thing", "--max-results=1", "-1", "--exec=cat"]); +} + +/// Filenames with non-utf8 paths are passed to the executed program unchanged +/// +/// Note: +/// - the test is disabled on Darwin/OSX, since it coerces file names to UTF-8, +/// even when the requested file name is not valid UTF-8. +/// - the test is currently disabled on Windows because I'm not sure how to create +/// invalid UTF-8 files on Windows +#[cfg(all(unix, not(target_os = "macos")))] +#[test] +fn test_exec_invalid_utf8() { + use std::ffi::OsStr; + use std::os::unix::ffi::OsStrExt; + + let dirs = &["test1"]; + let files = &[]; + let te = TestEnv::new(dirs, files); + + fs::File::create( + te.test_root() + .join(OsStr::from_bytes(b"test1/test_\xFEinvalid.txt")), + ) + .unwrap(); + + te.assert_output_raw( + &["", "test1/", "--exec", "echo", "{}"], + b"test1/test_\xFEinvalid.txt\n", + ); + + te.assert_output_raw( + &["", "test1/", "--exec", "echo", "{/}"], + b"test_\xFEinvalid.txt\n", + ); + + te.assert_output_raw(&["", "test1/", "--exec", "echo", "{//}"], b"test1\n"); + + te.assert_output_raw( + &["", "test1/", "--exec", "echo", "{.}"], + b"test1/test_\xFEinvalid\n", + ); + + te.assert_output_raw( + &["", "test1/", "--exec", "echo", "{/.}"], + b"test_\xFEinvalid\n", + ); +} + +#[test] +fn test_list_details() { + let te = TestEnv::new(DEFAULT_DIRS, DEFAULT_FILES); + + // On Windows, 'fd --list-details' needs GNU 'ls'. + if cfg!(windows) + && std::process::Command::new("ls") + .arg("--version") + .output() + .is_err() + { + te.assert_failure_with_error( + &["--list-details"], + "[fd error]: 'fd --list-details' is not supported on Windows unless GNU 'ls' is installed.", + ); + return; + } + + // Make sure we can execute 'fd --list-details' without any errors. + te.assert_success_and_get_output(".", &["--list-details"]); +} + +#[test] +fn test_single_and_multithreaded_execution() { + let te = TestEnv::new(DEFAULT_DIRS, DEFAULT_FILES); + + te.assert_output(&["--threads=1", "a.foo"], "a.foo"); + te.assert_output(&["--threads=16", "a.foo"], "a.foo"); +} + +/// Make sure that fd fails if numeric arguments can not be parsed +#[test] +fn test_number_parsing_errors() { + let te = TestEnv::new(&[], &[]); + + te.assert_failure(&["--threads=a"]); + te.assert_failure(&["-j", ""]); + te.assert_failure(&["--threads=0"]); + + te.assert_failure(&["--min-depth=a"]); + te.assert_failure(&["--mindepth=a"]); + te.assert_failure(&["--max-depth=a"]); + te.assert_failure(&["--maxdepth=a"]); + te.assert_failure(&["--exact-depth=a"]); + + te.assert_failure(&["--max-buffer-time=a"]); + + te.assert_failure(&["--max-results=a"]); +} + +#[test_case("--hidden", &["--no-hidden"] ; "hidden")] +#[test_case("--no-ignore", &["--ignore"] ; "no-ignore")] +#[test_case("--no-ignore-vcs", &["--ignore-vcs"] ; "no-ignore-vcs")] +#[test_case("--no-require-git", &["--require-git"] ; "no-require-git")] +#[test_case("--follow", &["--no-follow"] ; "follow")] +#[test_case("--absolute-path", &["--relative-path"] ; "absolute-path")] +#[test_case("-u", &["--ignore", "--no-hidden"] ; "u")] +#[test_case("-uu", &["--ignore", "--no-hidden"] ; "uu")] +fn test_opposing(flag: &str, opposing_flags: &[&str]) { + let te = TestEnv::new(DEFAULT_DIRS, DEFAULT_FILES); + + let mut flags = vec![flag]; + flags.extend_from_slice(opposing_flags); + let out_no_flags = te.assert_success_and_get_normalized_output(".", &[]); + let out_opposing_flags = te.assert_success_and_get_normalized_output(".", &flags); + + assert_eq!( + out_no_flags, + out_opposing_flags, + "{} should override {}", + opposing_flags.join(" "), + flag + ); +} + +#[test_case(&["-u", "--ignore"], &["--hidden"] ; "u ignore")] +#[test_case(&["-u", "--no-hidden"], &["--no-ignore"] ; "u no-hidden")] +#[test_case(&["--no-hidden", "-u"], &["--no-ignore"] ; "no-hidden u")] +fn test_unrestricted_partial_override(flags: &[&str], expected_flags: &[&str]) { + let te = TestEnv::new(DEFAULT_DIRS, DEFAULT_FILES); + + assert_eq!( + te.assert_success_and_get_normalized_output(".", flags), + te.assert_success_and_get_normalized_output(".", expected_flags), + ); +} + +/// Print error if search pattern starts with a dot and --hidden is not set +/// (Unix only, hidden files on Windows work differently) +#[test] +#[cfg(unix)] +fn test_error_if_hidden_not_set_and_pattern_starts_with_dot() { + let te = TestEnv::new(&[], &[".gitignore", ".whatever", "non-hidden"]); + + te.assert_failure(&["^\\.gitignore"]); + te.assert_failure(&["--glob", ".gitignore"]); + + te.assert_output(&["--hidden", "^\\.gitignore"], ".gitignore"); + te.assert_output(&["--hidden", "--glob", ".gitignore"], ".gitignore"); + te.assert_output(&[".gitignore"], ""); +} + +#[test] +fn test_strip_cwd_prefix() { + let te = TestEnv::new(DEFAULT_DIRS, DEFAULT_FILES); + + te.assert_output( + &["--strip-cwd-prefix", "."], + "a.foo + e1 e2 + one/ + one/b.foo + one/two/ + one/two/c.foo + one/two/C.Foo2 + one/two/three/ + one/two/three/d.foo + one/two/three/directory_foo/ + symlink", + ); +} + +/// When fd is ran from a non-existent working directory, but an existent +/// directory is passed in the arguments, it should still run fine +#[test] +#[cfg(all(not(windows), not(target_os = "illumos")))] +fn test_invalid_cwd() { + let te = TestEnv::new(&[], &[]); + + let root = te.test_root().join("foo"); + fs::create_dir(&root).unwrap(); + std::env::set_current_dir(&root).unwrap(); + fs::remove_dir(&root).unwrap(); + + let output = std::process::Command::new(te.test_exe()) + .arg("query") + .arg(te.test_root()) + .output() + .unwrap(); + + if !output.status.success() { + panic!("{output:?}"); + } +} + +/// Test behavior of .git directory with various flags +#[test] +fn test_git_dir() { + let te = TestEnv::new( + &[".git/one", "other_dir/.git", "nested/dir/.git"], + &[ + ".git/one/foo.a", + ".git/.foo", + ".git/a.foo", + "other_dir/.git/foo1", + "nested/dir/.git/foo2", + ], + ); + + te.assert_output( + &["--hidden", "foo"], + ".git/one/foo.a + .git/.foo + .git/a.foo + other_dir/.git/foo1 + nested/dir/.git/foo2", + ); + te.assert_output(&["--no-ignore", "foo"], ""); + te.assert_output( + &["--hidden", "--no-ignore", "foo"], + ".git/one/foo.a + .git/.foo + .git/a.foo + other_dir/.git/foo1 + nested/dir/.git/foo2", + ); + te.assert_output( + &["--hidden", "--no-ignore-vcs", "foo"], + ".git/one/foo.a + .git/.foo + .git/a.foo + other_dir/.git/foo1 + nested/dir/.git/foo2", + ); +} + +#[test] +fn test_gitignore_parent() { + let te = TestEnv::new(&["sub"], &[".abc", "sub/.abc"]); + + fs::File::create(te.test_root().join(".gitignore")) + .unwrap() + .write_all(b".abc\n") + .unwrap(); + + te.assert_output_subdirectory("sub", &["--hidden"], ""); + te.assert_output_subdirectory("sub", &["--hidden", "--search-path", "."], ""); +} + +#[test] +fn test_hyperlink() { + let te = TestEnv::new(DEFAULT_DIRS, DEFAULT_FILES); + + #[cfg(unix)] + let hostname = nix::unistd::gethostname().unwrap().into_string().unwrap(); + #[cfg(not(unix))] + let hostname = "/"; + + let expected = format!( + "\x1b]8;;file://{}{}/a.foo\x1b\\a.foo\x1b]8;;\x1b\\", + hostname, + get_absolute_root_path(&te), + ); + + te.assert_output(&["--hyperlink=always", "a.foo"], &expected); +} + +#[test] +fn test_ignore_contain() { + let te = TestEnv::new( + &["include", "exclude", "exclude/sub", "other"], + &[ + "top", + "include/foo", + "exclude/CACHEDIR.TAG", + "exclude/sub/nope", + "other/ignoremyparent", + ], + ); + let expected = "include/ + include/foo + symlink + top"; + te.assert_output( + &[ + "--ignore-contain=CACHEDIR.TAG", + "--ignore-contain=ignoremyparent", + ".", + ], + expected, + ); +} + +#[test] +fn test_ignore_contain_precedence_over_depth_check() { + let te = TestEnv::new( + &["include", "exclude", "exclude/sub"], + &[ + "top", + "include/foo", + "exclude/CACHEDIR.TAG", + "exclude/sub/nope", + ], + ); + let expected = "include/foo"; + te.assert_output( + &["--ignore-contain=CACHEDIR.TAG", "--min-depth=2", "."], + expected, + ); +} + +#[test] +fn test_ignore_contain_precedence_over_root_check() { + let te = TestEnv::new(&["include"], &["CACHEDIR.TAG", "top", "include/foo"]); + let expected = ""; + te.assert_output(&["--ignore-contain=CACHEDIR.TAG", "."], expected); +} + +// The error message is probably OS-specific. +// This is also somewhat fragile as it depends on the error message +// from creating an executable, and the debug formatting of strings +// could possibly change in the future. +#[cfg(unix)] +#[test] +fn test_sanitize_exec_error_msg() { + let mut te = TestEnv::new(&[], &[]); + te.create_broken_symlink("Hello\x1b\r World!\x7fwith\u{9b}\u{1F600}\u{200B}a\u{FEFF}b") + .expect("failed to create symlink"); + + te.assert_error(&["Hello", "--exec", "{}"], + "[fd error]: Command not found: \"./Hello\\u{1b}\\r World!\\u{7f}with\\u{9b}\u{1F600}\\u{200b}a\\u{feff}b\"" + ); +} + +// Windows doesn't let us make files containing \x1b so only test on unix +#[cfg(unix)] +#[test] +fn test_sanitize_recursive_link_msg() { + let mut te = TestEnv::new(&["loop"], &[]); + te.create_symlink("loop", "loop/foo\x1bb") + .expect("failed to create symlink"); + + te.assert_error( + &["--follow", "--show-errors", "foo"], + "[fd error]: File system loop found: ./loop/foo\\x1Bb points to an ancestor ./loop", + ); +} diff --git a/userland/findutils/Cargo.toml b/userland/findutils/Cargo.toml new file mode 100644 index 0000000..c49b94c --- /dev/null +++ b/userland/findutils/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "sausage-findutils" +version = "0.1.0" +edition = "2024" + +[[bin]] +name = "find" +path = "src/find.rs" + +[[bin]] +name = "xargs" +path = "src/xargs.rs" + +[dependencies] +walkdir = "=2.5.0" +globset = "=0.4.19" +sausage-spawn = { path = "../spawn" } +shlex = "=1.3.0" diff --git a/userland/findutils/src/find.rs b/userland/findutils/src/find.rs new file mode 100644 index 0000000..4c8b507 --- /dev/null +++ b/userland/findutils/src/find.rs @@ -0,0 +1,67 @@ +//! Find world paths by name, type, path, depth, or modification time. +use std::{io::{self, Write}, path::PathBuf}; +use sausage_spawn::Command; + +enum Predicate { Name(globset::GlobMatcher), Path(globset::GlobMatcher), Type(String), Newer(std::time::SystemTime) } +fn run() -> Result> { + let args: Vec<_> = std::env::args().skip(1).collect(); + let mut i = 0; + let mut roots = Vec::new(); + while i < args.len() && !args[i].starts_with('-') { roots.push(PathBuf::from(&args[i])); i += 1; } + if roots.is_empty() { roots.push(".".into()); } + let mut predicates = Vec::new(); + let mut maxdepth = usize::MAX; + let mut null = false; + let mut exec = None; + while i < args.len() { + let flag = &args[i]; i += 1; + match flag.as_str() { + "-print" => (), "-print0" => null = true, + "-name" | "-path" | "-type" | "-maxdepth" | "-newer" => { + let value = args.get(i).ok_or("missing predicate argument")?; i += 1; + match flag.as_str() { + "-name" => predicates.push(Predicate::Name(globset::Glob::new(value)?.compile_matcher())), + "-path" => predicates.push(Predicate::Path(globset::Glob::new(value)?.compile_matcher())), + "-type" => predicates.push(Predicate::Type(value.clone())), + "-newer" => predicates.push(Predicate::Newer(std::fs::metadata(value)?.modified()?)), + _ => maxdepth = value.parse()?, + } + } + "-exec" => { + let start = i; + while i < args.len() && args[i] != ";" { i += 1; } + if i == args.len() || i == start { return Err("-exec requires a command and ;".into()); } + exec = Some(args[start..i].to_vec()); i += 1; + } + _ => return Err(format!("unsupported predicate: {flag}").into()), + } + } + let mut status = 0; + let mut out = io::BufWriter::new(io::stdout().lock()); + for root in roots { + for entry in walkdir::WalkDir::new(root).max_depth(maxdepth) { + let entry = match entry { Ok(e) => e, Err(e) => { eprintln!("find: {e}"); status = 1; continue; } }; + let mut matches = true; + for predicate in &predicates { + matches &= match predicate { + Predicate::Name(g) => g.is_match(entry.file_name()), + Predicate::Path(g) => g.is_match(entry.path()), + Predicate::Type(t) => match t.as_str() { "f" => entry.file_type().is_file(), "d" => entry.file_type().is_dir(), "l" => entry.file_type().is_symlink(), _ => return Err("unsupported file type".into()) }, + Predicate::Newer(time) => entry.metadata()?.modified()? > *time, + }; + } + if !matches { continue; } + if let Some(argv) = &exec { + out.flush()?; + let argv: Vec<_> = argv.iter().map(|arg| arg.replace("{}", &entry.path().to_string_lossy())).collect(); + if let Err(e) = Command::new(&argv[0]).args(&argv[1..]).status() { eprintln!("find: {e}"); status = 1; } + } else { + out.write_all(entry.path().as_os_str().as_encoded_bytes())?; + out.write_all(if null { b"\0" } else { b"\n" })?; + } + } + } + out.flush()?; + Ok(status) +} +fn main() { std::process::exit(match run() { Ok(code) => code, Err(e) => { eprintln!("find: {e}"); 1 } }); } diff --git a/userland/findutils/src/xargs.rs b/userland/findutils/src/xargs.rs new file mode 100644 index 0000000..fbc43b2 --- /dev/null +++ b/userland/findutils/src/xargs.rs @@ -0,0 +1,39 @@ +//! Build sequential child commands from quoted or NUL-delimited input. +use std::io::{self, Read}; +use sausage_spawn::{Command, Stdio}; +fn run() -> Result> { + let args: Vec<_> = std::env::args().skip(1).collect(); + let mut i = 0; + let mut count = usize::MAX; + let mut null = false; + let mut no_empty = false; + while i < args.len() { + match args[i].as_str() { + "-0" => null = true, + "-r" => no_empty = true, + "-n" => { i += 1; count = args.get(i).ok_or("missing -n argument")?.parse()?; } + "--" => { i += 1; break; } + arg if arg.starts_with("-n") => count = arg[2..].parse()?, + arg if arg.starts_with('-') => return Err(format!("unsupported option: {arg}").into()), + _ => break, + } + i += 1; + } + if count == 0 { return Err("-n must be positive".into()); } + let argv = if i == args.len() { vec!["echo".into()] } else { args[i..].to_vec() }; + let mut input = String::new(); io::stdin().read_to_string(&mut input)?; + let values = if null { input.split_terminator('\0').map(str::to_owned).collect() } + else { shlex::split(&input).ok_or("unmatched quote in input")? }; + if values.is_empty() && no_empty { return Ok(0); } + let mut result = 0; + let batches = if values.is_empty() { vec![&[][..]] } else { values.chunks(count).collect() }; + for batch in batches { + match Command::new(&argv[0]).args(&argv[1..]).args(batch).stdin(Stdio::Null).status() { + Ok(0) => (), Ok(255) => return Ok(124), Ok(_) => result = 123, + Err(e) if e.kind() == io::ErrorKind::NotFound => return Ok(127), + Err(_) => return Ok(126), + } + } + Ok(result) +} +fn main() { std::process::exit(match run() { Ok(code) => code, Err(e) => { eprintln!("xargs: {e}"); 1 } }); } diff --git a/userland/grep/Cargo.toml b/userland/grep/Cargo.toml new file mode 100644 index 0000000..6dcd20d --- /dev/null +++ b/userland/grep/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "sausage-grep" +version = "0.1.0" +edition = "2024" + +[[bin]] +name = "grep" +path = "src/main.rs" + +[dependencies] +grep-regex = "=0.1.14" +grep-matcher = "=0.1.9" +lexopt = "=0.3.2" +globset = "=0.4.19" +walkdir = "=2.5.0" diff --git a/userland/grep/src/main.rs b/userland/grep/src/main.rs new file mode 100644 index 0000000..9c5b496 --- /dev/null +++ b/userland/grep/src/main.rs @@ -0,0 +1,158 @@ +//! GNU-style line search on ripgrep's regex engine. +use grep_matcher::Matcher; +use grep_regex::RegexMatcherBuilder; +use std::io::{self, Read, Write}; +use std::path::{Path, PathBuf}; + +#[derive(Default)] +struct Options { + patterns: Vec, files: Vec, recursive: bool, + number: bool, ignore_case: bool, invert: bool, list: bool, list_without: bool, + count: bool, fixed: bool, extended: bool, word: bool, whole: bool, + only: bool, quiet: bool, silent: bool, filename: Option, + before: usize, after: usize, + include: Vec, exclude: Vec, + exclude_dir: Vec, +} +fn parse() -> Result> { + use lexopt::Arg::*; + let mut p = lexopt::Parser::from_env(); + let mut o = Options::default(); + while let Some(arg) = p.next()? { + match arg { + Short('r' | 'R') => o.recursive = true, + Short('n') => o.number = true, + Short('i') => o.ignore_case = true, + Short('v') => o.invert = true, + Short('l') => o.list = true, + Short('L') => o.list_without = true, + Short('c') => o.count = true, + Short('E') => o.extended = true, + Short('G') => o.extended = false, + Short('F') => o.fixed = true, + Short('w') => o.word = true, + Short('x') => o.whole = true, + Short('o') => o.only = true, + Short('q') => o.quiet = true, + Short('s') => o.silent = true, + Short('h') => o.filename = Some(false), + Short('H') => o.filename = Some(true), + Short('A') => o.after = p.value()?.to_string_lossy().parse()?, + Short('B') => o.before = p.value()?.to_string_lossy().parse()?, + Short('C') => { let n = p.value()?.to_string_lossy().parse()?; o.before = n; o.after = n; } + Short('e') => o.patterns.push(p.value()?.to_string_lossy().into_owned()), + Short('f') => { + let file = p.value()?; + let text = std::fs::read_to_string(file)?; + o.patterns.extend(text.lines().map(str::to_owned)); + } + Long("include" | "exclude" | "exclude-dir") => { + let name = match arg { Long(name) => name.to_owned(), _ => unreachable!() }; + let g = globset::Glob::new(&p.value()?.to_string_lossy())?.compile_matcher(); + match name.as_str() { "include" => o.include.push(g), "exclude" => o.exclude.push(g), _ => o.exclude_dir.push(g) } + } + Value(v) => { + if o.patterns.is_empty() { o.patterns.push(v.to_string_lossy().into_owned()); } + else { o.files.push(v.into()); } + } + _ => return Err(arg.unexpected().into()), + } + } + if o.patterns.is_empty() { return Err("a pattern is required".into()); } + if o.files.is_empty() { o.files.push(if o.recursive { ".".into() } else { "-".into() }); } + Ok(o) +} +// GNU basic regex quotes operators that extended regex treats as operators. +fn basic(pattern: &str) -> String { + let mut out = String::new(); + let mut chars = pattern.chars(); + while let Some(c) = chars.next() { + if c == '\\' { + if let Some(n) = chars.next() { + if "()+?|{}".contains(n) { out.push(n); } + else { out.push('\\'); out.push(n); } + } else { out.push('\\'); } + } else { + if "()+?|{}".contains(c) { out.push('\\'); } + out.push(c); + } + } + out +} +fn allowed(path: &Path, o: &Options) -> bool { + let name = path.file_name().unwrap_or(path.as_os_str()); + (o.include.is_empty() || o.include.iter().any(|g| g.is_match(name))) + && !o.exclude.iter().any(|g| g.is_match(name)) +} +fn run(o: Options) -> Result> { + let patterns: Vec<_> = o.patterns.iter().map(|p| if o.fixed || o.extended { p.clone() } else { basic(p) }).collect(); + let matcher = RegexMatcherBuilder::new().case_insensitive(o.ignore_case) + .fixed_strings(o.fixed).word(o.word).whole_line(o.whole).build_many(&patterns)?; + let names = o.filename.unwrap_or(o.files.len() > 1 || o.recursive); + let mut files = Vec::new(); + let mut error = false; + for path in &o.files { + if o.recursive && path.is_dir() { + for entry in walkdir::WalkDir::new(path).follow_links(true).into_iter().filter_entry(|e| { + !e.file_type().is_dir() || !o.exclude_dir.iter().any(|g| g.is_match(e.file_name())) + }) { + match entry { + Ok(e) if e.file_type().is_file() && allowed(e.path(), &o) => files.push(e.into_path()), + Ok(_) => (), + Err(e) => { error = true; if !o.silent { eprintln!("grep: {e}"); } } + } + } + } else if allowed(path, &o) { files.push(path.clone()); } + } + let mut matched = false; + let mut out = io::BufWriter::new(io::stdout().lock()); + for path in files { + let mut data = Vec::new(); + let read = if path == Path::new("-") { io::stdin().read_to_end(&mut data) } + else { std::fs::File::open(&path).and_then(|mut f| f.read_to_end(&mut data)) }; + if let Err(e) = read { error = true; if !o.silent { eprintln!("grep: {}: {e}", path.display()); } continue; } + let lines: Vec<_> = data.split_inclusive(|b| *b == b'\n').collect(); + let hits: Vec<_> = lines.iter().map(|line| { + let line = line.strip_suffix(b"\n").unwrap_or(line); + matcher.is_match(line).map(|m| m != o.invert) + }).collect::>()?; + let count = hits.iter().filter(|v| **v).count(); + matched |= count > 0; + if o.quiet && count > 0 { return Ok(0); } + let label = if path == Path::new("-") { "(standard input)".into() } else { path.to_string_lossy() }; + if o.list || o.list_without { + if (o.list && count > 0) || (o.list_without && count == 0) { writeln!(out, "{label}")?; } + } else if o.count { + if names { write!(out, "{label}:")?; } + writeln!(out, "{count}")?; + } else if !o.quiet { + let context = o.before > 0 || o.after > 0; + let mut last = None; + for (i, line) in lines.iter().enumerate() { + let selected = hits[i]; + let visible = selected || (context && (i.saturating_sub(o.after)..=(i + o.before).min(lines.len().saturating_sub(1))).any(|j| hits[j])); + if !visible { continue; } + if o.only && !selected { continue; } + if context && last.is_some_and(|last| i > last + 1) { writeln!(out, "--")?; } + last = Some(i); + let sep = if selected { ':' } else { '-' }; + let mut emit = |bytes: &[u8]| -> io::Result<()> { + if names { write!(out, "{label}{sep}")?; } + if o.number { write!(out, "{}{sep}", i + 1)?; } + out.write_all(bytes)?; + if !bytes.ends_with(b"\n") { writeln!(out)?; } + Ok(()) + }; + if o.only { + if !o.invert { matcher.try_find_iter(line, |m| { if !m.is_empty() { emit(&line[m.start()..m.end()])?; } Ok::<_, io::Error>(true) })??; } + } else { emit(line)?; } + } + } + } + out.flush()?; + Ok(if error { 2 } else if matched { 0 } else { 1 }) +} +fn main() { + let code = match parse().and_then(run) { Ok(code) => code, Err(e) => { eprintln!("grep: {e}"); 2 } }; + std::process::exit(code); +} diff --git a/userland/jaq/.cargo-ok b/userland/jaq/.cargo-ok new file mode 100644 index 0000000..5f8b795 --- /dev/null +++ b/userland/jaq/.cargo-ok @@ -0,0 +1 @@ +{"v":1} \ No newline at end of file diff --git a/userland/jaq/.cargo_vcs_info.json b/userland/jaq/.cargo_vcs_info.json new file mode 100644 index 0000000..00dd06f --- /dev/null +++ b/userland/jaq/.cargo_vcs_info.json @@ -0,0 +1,6 @@ +{ + "git": { + "sha1": "0ce6e86a5e038a623dc894ad5cc70aaa9142daf2" + }, + "path_in_vcs": "jaq" +} \ No newline at end of file diff --git a/userland/jaq/Cargo.lock b/userland/jaq/Cargo.lock new file mode 100644 index 0000000..04d0d89 --- /dev/null +++ b/userland/jaq/Cargo.lock @@ -0,0 +1,845 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "aho-corasick" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" +dependencies = [ + "memchr", +] + +[[package]] +name = "android-tzdata" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0" + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + +[[package]] +name = "autocfg" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "bitflags" +version = "2.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b8e56985ec62d17e9c1001dc89c88ecd7dc08e47eba5ec7c29c7b5eeecde967" + +[[package]] +name = "bumpalo" +version = "3.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f30e7476521f6f8af1a1c4c0b8cc94f0bee37d91763d0ca2665f299b6cd8aec" + +[[package]] +name = "cc" +version = "1.2.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c1599538de2394445747c8cf7935946e3cc27e9625f889d979bfb2aaf569362" +dependencies = [ + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9555578bc9e57714c812a1f84e4fc5b4d21fcb063490c624de019f7464c91268" + +[[package]] +name = "chrono" +version = "0.4.41" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c469d952047f47f91b68d1cba3f10d63c11d73e4636f24f08daf0278abf01c4d" +dependencies = [ + "android-tzdata", + "iana-time-zone", + "num-traits", + "windows-link", +] + +[[package]] +name = "clipboard-win" +version = "5.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15efe7a882b08f34e38556b14f2fb3daa98769d06c7f0c1b076dfd0d983bc892" +dependencies = [ + "error-code", +] + +[[package]] +name = "codesnake" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2205f7f6d3de68ecf4c291c789b3edf07b6569268abd0188819086f71ae42225" + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "dirs" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3e8aa94d75141228480295a7d0e7feb620b1a5ad9f12bc40be62411e38cce4e" +dependencies = [ + "dirs-sys", +] + +[[package]] +name = "dirs-sys" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e01a3366d27ee9890022452ee61b2b63a67e6f13f58900b651ff5665f0bb1fab" +dependencies = [ + "libc", + "option-ext", + "redox_users", + "windows-sys", +] + +[[package]] +name = "dyn-clone" +version = "1.0.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c7a8fb8a9fbf66c1f703fe16184d10ca0ee9d23be5b4436400408ba54a95005" + +[[package]] +name = "env_logger" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cd405aab171cb85d6735e5c8d9db038c17d3ca007a4d2c25f337935c3d90580" +dependencies = [ + "log", +] + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "errno" +version = "0.3.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "778e2ac28f6c47af28e4907f13ffd1e1ddbd400980a9abd7c8df189bf578a5ad" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "error-code" +version = "3.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dea2df4cf52843e0452895c455a1a2cfbb842a1e7329671acf418fdc53ed4c59" + +[[package]] +name = "fastrand" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" + +[[package]] +name = "fd-lock" +version = "4.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ce92ff622d6dadf7349484f42c93271a0d49b7cc4d466a936405bacbe10aa78" +dependencies = [ + "cfg-if", + "rustix", + "windows-sys", +] + +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + +[[package]] +name = "getrandom" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "335ff9f135e4384c8150d6f27c6daed433577f86b4750418338c01a1a2528592" +dependencies = [ + "cfg-if", + "libc", + "wasi 0.11.1+wasi-snapshot-preview1", +] + +[[package]] +name = "getrandom" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26145e563e54f2cadc477553f1ec5ee650b00862f0a58bcd12cbdc5f0ea2d2f4" +dependencies = [ + "cfg-if", + "libc", + "r-efi", + "wasi 0.14.2+wasi-0.2.4", +] + +[[package]] +name = "hashbrown" +version = "0.15.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5971ac85611da7067dbfcabef3c70ebb5606018acd9e2a3903a0da507521e0d5" + +[[package]] +name = "hermit-abi" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" + +[[package]] +name = "hifijson" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9958ab3ce3170c061a27679916bd9b969eceeb5e8b120438e6751d0987655c42" + +[[package]] +name = "iana-time-zone" +version = "0.1.63" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0c919e5debc312ad217002b8048a17b7d83f80703865bbfcfebb0458b0b27d8" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "indexmap" +version = "2.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe4cd85333e22411419a0bcae1297d25e58c9443848b11dc6a86fefe8c78a661" +dependencies = [ + "equivalent", + "hashbrown", +] + +[[package]] +name = "is-terminal" +version = "0.4.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e04d7f318608d35d4b61ddd75cbdaee86b023ebe2bd5a66ee0915f0bf93095a9" +dependencies = [ + "hermit-abi", + "libc", + "windows-sys", +] + +[[package]] +name = "jaq" +version = "2.3.0" +dependencies = [ + "codesnake", + "dirs", + "env_logger", + "hifijson", + "is-terminal", + "jaq-core", + "jaq-json", + "jaq-std", + "log", + "memmap2", + "mimalloc", + "rustyline", + "tempfile", + "unicode-width", + "yansi", +] + +[[package]] +name = "jaq-core" +version = "2.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77526a72eb79412c29fd141767a6549bbfcb1cb40e00556fe16532d5e878e098" +dependencies = [ + "dyn-clone", + "once_cell", + "typed-arena", +] + +[[package]] +name = "jaq-json" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01dbdbd07b076e8403abac68ce7744d93e2ecd953bbc44bf77bf00e1e81172bc" +dependencies = [ + "foldhash", + "hifijson", + "indexmap", + "jaq-core", + "jaq-std", +] + +[[package]] +name = "jaq-std" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c264fe397c981705976c71f1bfe020382b9eda52ae950e57fe885e147bdd67d" +dependencies = [ + "aho-corasick", + "base64", + "chrono", + "jaq-core", + "libm", + "log", + "regex-lite", + "urlencoding", +] + +[[package]] +name = "js-sys" +version = "0.3.77" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1cfaf33c695fc6e08064efbc1f72ec937429614f25eef83af942d0e227c3a28f" +dependencies = [ + "once_cell", + "wasm-bindgen", +] + +[[package]] +name = "libc" +version = "0.2.174" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1171693293099992e19cddea4e8b849964e9846f4acee11b3948bcc337be8776" + +[[package]] +name = "libm" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9fbbcab51052fe104eb5e5d351cf728d30a5be1fe14d9be8a3b097481fb97de" + +[[package]] +name = "libmimalloc-sys" +version = "0.1.43" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf88cd67e9de251c1781dbe2f641a1a3ad66eaae831b8a2c38fbdc5ddae16d4d" +dependencies = [ + "cc", + "libc", +] + +[[package]] +name = "libredox" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1580801010e535496706ba011c15f8532df6b42297d2e471fec38ceadd8c0638" +dependencies = [ + "bitflags", + "libc", +] + +[[package]] +name = "linux-raw-sys" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd945864f07fe9f5371a27ad7b52a172b4b499999f1d97574c9fa68373937e12" + +[[package]] +name = "log" +version = "0.4.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13dc2df351e3202783a1fe0d44375f7295ffb4049267b0f3018346dc122a1d94" + +[[package]] +name = "memchr" +version = "2.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a282da65faaf38286cf3be983213fcf1d2e2a58700e808f83f4ea9a4804bc0" + +[[package]] +name = "memmap2" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd3f7eed9d3848f8b98834af67102b720745c4ec028fcd0aa0239277e7de374f" +dependencies = [ + "libc", +] + +[[package]] +name = "mimalloc" +version = "0.1.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1791cbe101e95af5764f06f20f6760521f7158f69dbf9d6baf941ee1bf6bc40" +dependencies = [ + "libmimalloc-sys", +] + +[[package]] +name = "nix" +version = "0.27.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2eb04e9c688eff1c89d72b407f168cf79bb9e867a9d3323ed6c01519eb9cc053" +dependencies = [ + "bitflags", + "cfg-if", + "libc", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "once_cell" +version = "1.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "945462a4b81e43c4e3ba96bd7b49d834c6f61198356aa858733bc4acf3cbe62e" + +[[package]] +name = "option-ext" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" + +[[package]] +name = "proc-macro2" +version = "1.0.95" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02b3e5e68a3a1a02aad3ec490a98007cbc13c37cbe84a3cd7b8e406d76e7f778" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1885c039570dc00dcb4ff087a89e185fd56bae234ddc7f056a945bf36467248d" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "redox_users" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd6f9d3d47bdd2ad6945c5015a226ec6155d0bcdfd8f7cd29f86b71f8de99d2b" +dependencies = [ + "getrandom 0.2.16", + "libredox", + "thiserror", +] + +[[package]] +name = "regex-lite" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53a49587ad06b26609c52e423de037e7f57f20d53535d66e08c695f347df952a" + +[[package]] +name = "rustix" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c71e83d6afe7ff64890ec6b71d6a69bb8a610ab78ce364b3352876bb4c801266" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys", +] + +[[package]] +name = "rustversion" +version = "1.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a0d197bd2c9dc6e53b84da9556a69ba4cdfab8619eb41a8bd1cc2027a0f6b1d" + +[[package]] +name = "rustyline" +version = "13.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02a2d683a4ac90aeef5b1013933f6d977bd37d51ff3f4dad829d4931a7e6be86" +dependencies = [ + "bitflags", + "cfg-if", + "clipboard-win", + "fd-lock", + "libc", + "log", + "memchr", + "nix", + "unicode-segmentation", + "unicode-width", + "utf8parse", + "winapi", +] + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + +[[package]] +name = "syn" +version = "2.0.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17b6f705963418cdb9927482fa304bc562ece2fdd4f616084c50b7023b435a40" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "tempfile" +version = "3.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8a64e3985349f2441a1a9ef0b853f869006c3855f2cda6862a94d26ebb9d6a1" +dependencies = [ + "fastrand", + "getrandom 0.3.3", + "once_cell", + "rustix", + "windows-sys", +] + +[[package]] +name = "thiserror" +version = "2.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "567b8a2dae586314f7be2a752ec7474332959c6460e02bde30d702a66d488708" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f7cf42b4507d8ea322120659672cf1b9dbb93f8f2d4ecfd6e51350ff5b17a1d" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "typed-arena" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6af6ae20167a9ece4bcb41af5b80f8a1f1df981f6391189ce00fd257af04126a" + +[[package]] +name = "unicode-ident" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a5f39404a5da50712a4c1eecf25e90dd62b613502b7e925fd4e4d19b5c96512" + +[[package]] +name = "unicode-segmentation" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493" + +[[package]] +name = "unicode-width" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af" + +[[package]] +name = "urlencoding" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" + +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasi" +version = "0.14.2+wasi-0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9683f9a5a998d873c0d21fcbe3c083009670149a8fab228644b8bd36b2c48cb3" +dependencies = [ + "wit-bindgen-rt", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1edc8929d7499fc4e8f0be2262a241556cfc54a0bea223790e71446f2aab1ef5" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f0a0651a5c2bc21487bde11ee802ccaf4c51935d0d3d42a6101f98161700bc6" +dependencies = [ + "bumpalo", + "log", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fe63fc6d09ed3792bd0897b314f53de8e16568c2b3f7982f468c0bf9bd0b407" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ae87ea40c9f689fc23f209965b6fb8a99ad69aeeb0231408be24920604395de" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.100" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a05d73b933a847d6cccdda8f838a22ff101ad9bf93e33684f39c1f5f0eece3d" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-core" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0fdd3ddb90610c7638aa2b3a3ab2904fb9e5cdbecc643ddb3647212781c4ae3" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-implement" +version = "0.60.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a47fddd13af08290e67f4acabf4b459f647552718f683a7b415d290ac744a836" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-interface" +version = "0.59.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd9211b69f8dcdfa817bfd14bf1c97c9188afa36f4750130fcdf3f400eca9fa8" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-link" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e6ad25900d524eaabdbbb96d20b4311e1e7ae1699af4fb28c17ae66c80d798a" + +[[package]] +name = "windows-result" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56f42bd332cc6c8eac5af113fc0c1fd6a8fd2aa08a0119358686e5160d0586c6" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56e6c93f3a0c3b36176cb1327a4958a0353d5d166c2a35cb268ace15e91d3b57" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "wit-bindgen-rt" +version = "0.39.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f42320e61fe2cfd34354ecb597f86f413484a798ba44a8ca1165c58d42da6c1" +dependencies = [ + "bitflags", +] + +[[package]] +name = "yansi" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfe53a6657fd280eaa890a3bc59152892ffa3e30101319d168b781ed6529b049" diff --git a/userland/jaq/Cargo.toml b/userland/jaq/Cargo.toml new file mode 100644 index 0000000..9c9db9f --- /dev/null +++ b/userland/jaq/Cargo.toml @@ -0,0 +1,100 @@ +# THIS FILE IS AUTOMATICALLY GENERATED BY CARGO +# +# When uploading crates to the registry Cargo will automatically +# "normalize" Cargo.toml files for maximal compatibility +# with all versions of Cargo and also rewrite `path` dependencies +# to registry (e.g., crates.io) dependencies. +# +# If you are reading this file be aware that the original Cargo.toml +# will likely look very different (and much more reasonable). +# See Cargo.toml.orig for the original contents. + +[package] +edition = "2021" +rust-version = "1.66" +name = "jaq" +version = "2.3.0" +authors = ["Michael Färber "] +build = false +autolib = false +autobins = false +autoexamples = false +autotests = false +autobenches = false +description = "Just another JSON query tool" +readme = "README.md" +keywords = [ + "json", + "query", + "jq", +] +categories = [ + "command-line-utilities", + "compilers", + "parser-implementations", +] +license = "MIT" +repository = "https://github.com/01mf02/jaq" + +[features] +# WASI uses the standard allocator, not native mimalloc. +default = [] + +[[bin]] +name = "jaq" +path = "src/main.rs" + +[[test]] +name = "golden" +path = "tests/golden.rs" + +[dependencies.codesnake] +version = "0.2" + +[dependencies.dirs] +version = "6.0" + +[dependencies.env_logger] +version = "0.10.0" +default-features = false + +[dependencies.hifijson] +version = "0.2.0" + +[dependencies.is-terminal] +version = "0.4.13" + +[dependencies.jaq-core] +version = "2.1.1" + +[dependencies.jaq-json] +version = "1.1.1" + +[dependencies.jaq-std] +version = "2.1.0" + +[dependencies.log] +version = "0.4.17" + +[dependencies.memmap2] +version = "0.9" + +[dependencies.mimalloc] +version = "0.1.29" +optional = true +default-features = false + +# WASI has no terminal REPL or file locks. +[target.'cfg(not(target_os = "wasi"))'.dependencies.rustyline] +version = "13.0.0" +features = ["with-file-history"] +default-features = false + +[dependencies.tempfile] +version = "3.3.0" + +[dependencies.unicode-width] +version = "0.1.13" + +[dependencies.yansi] +version = "1.0.1" diff --git a/userland/jaq/Cargo.toml.orig b/userland/jaq/Cargo.toml.orig new file mode 100644 index 0000000..e84bb75 --- /dev/null +++ b/userland/jaq/Cargo.toml.orig @@ -0,0 +1,33 @@ +[package] +name = "jaq" +version = "2.3.0" +authors = ["Michael Färber "] +edition = "2021" +license = "MIT" +readme = "../README.md" +description = "Just another JSON query tool" +repository = "https://github.com/01mf02/jaq" +keywords = ["json", "query", "jq"] +categories = ["command-line-utilities", "compilers", "parser-implementations"] +rust-version = "1.66" + +[features] +default = ["mimalloc"] + +[dependencies] +jaq-core = { version = "2.1.1", path = "../jaq-core" } +jaq-std = { version = "2.1.0", path = "../jaq-std" } +jaq-json = { version = "1.1.1", path = "../jaq-json" } + +codesnake = { version = "0.2" } +dirs = { version = "6.0" } +env_logger = { version = "0.10.0", default-features = false } +hifijson = "0.2.0" +is-terminal = "0.4.13" +log = { version = "0.4.17" } +memmap2 = "0.9" +mimalloc = { version = "0.1.29", default-features = false, optional = true } +rustyline = { version = "13.0.0", default-features = false, features = ["with-file-history"] } +tempfile = "3.3.0" +unicode-width = "0.1.13" +yansi = "1.0.1" diff --git a/userland/jaq/PATCHES.md b/userland/jaq/PATCHES.md new file mode 100644 index 0000000..f7f9a7b --- /dev/null +++ b/userland/jaq/PATCHES.md @@ -0,0 +1,11 @@ +# WASI patches + +Source: crates.io `jaq` 2.3.0, https://github.com/01mf02/jaq. +The vendor copy retains the upstream license. + +- `Cargo.toml` disables the default mimalloc feature. +- `Cargo.toml` excludes rustyline on WASI because its file-lock dependency does not compile there. +- `src/main.rs` excludes the REPL module on WASI. +- `src/filter.rs` excludes the REPL function registration on WASI. + +The JSON parser, filter compiler, and filter execution retain their upstream behavior. diff --git a/userland/jaq/README.md b/userland/jaq/README.md new file mode 100644 index 0000000..c08220f --- /dev/null +++ b/userland/jaq/README.md @@ -0,0 +1,606 @@ +# jaq + +![Build status](https://github.com/01mf02/jaq/actions/workflows/check.yml/badge.svg) +[![Crates.io](https://img.shields.io/crates/v/jaq-core.svg)](https://crates.io/crates/jaq-core) +[![Documentation](https://docs.rs/jaq-core/badge.svg)](https://docs.rs/jaq-core) +[![Rust 1.66+](https://img.shields.io/badge/rust-1.66+-orange.svg)](https://www.rust-lang.org) + +jaq (pronounced /Ê’aËk/, like *Jacques*[^jacques]) is a clone of the JSON data processing tool [jq]. +jaq aims to support a large subset of jq's syntax and operations. + +You can try jaq online on the [jaq playground](https://gedenkt.at/jaq/). +Instructions for the playground can be found [here](jaq-play/). + +jaq focuses on three goals: + +* **Correctness**: + jaq aims to provide a more correct and predictable implementation of jq, + while preserving compatibility with jq in most cases. +* **Performance**: + I created jaq originally because I was bothered by + [the long start-up time of jq 1.6](https://github.com/jqlang/jq/issues/1411), + which amounts to about 50ms on my machine. + This can be particularly seen when processing a large number of small files. + Although the startup time has been vastly improved in jq 1.7, + jaq is still faster than jq on many other [benchmarks](#performance). +* **Simplicity**: + jaq aims to have a simple and small implementation, in order to + reduce the potential for bugs and to + facilitate contributions. + +I drew inspiration from another Rust program, namely [jql]. +However, unlike jql, jaq aims to closely imitate jq's syntax and semantics. +This should allow users proficient in jq to easily use jaq. + +[jq]: https://jqlang.github.io/jq/ +[jql]: https://github.com/yamafaktory/jql + +[^jacques]: I wanted to create a tool that should be discreet and obliging, like a good waiter. + And when I think of a typical name for a (French) waiter, to my mind comes "Jacques". + Later, I found out about the old French word *jacquet*, meaning "squirrel", + which makes for a nice *ex post* inspiration for the name. + + + +# Installation + + +## Binaries + +You can download binaries for Linux, Mac, and Windows on the [releases page](https://github.com/01mf02/jaq/releases). +On a Linux system, you can download it using the following commands: + + $ curl -fsSL https://github.com/01mf02/jaq/releases/latest/download/jaq-$(uname -m)-unknown-linux-musl -o jaq && chmod +x jaq + +You may also install jaq using [homebrew](https://formulae.brew.sh/formula/jaq) on macOS or Linux: + + $ brew install jaq + $ brew install --HEAD jaq # latest development version + +[![Packaging status](https://repology.org/badge/vertical-allrepos/jaq.svg)](https://repology.org/project/jaq/versions) + + +## From Source + +To compile jaq, you need a Rust toolchain. +See for instructions. + +Any of the following commands install jaq: + + $ cargo install --locked jaq + $ cargo install --locked --git https://github.com/01mf02/jaq # latest development version + +On my system, both commands place the executable at `~/.cargo/bin/jaq`. + +If you have cloned this repository, you can also build jaq by executing one of the commands in the cloned repository: + + $ cargo build --release # places binary into target/release/jaq + $ cargo install --locked --path jaq # installs binary + +jaq should work on any system supported by Rust. +If it does not, please file an issue. + + + +# Examples + +The following examples should give an impression of what jaq can currently do. +You should obtain the same outputs by replacing jaq with jq. +If not, your filing an issue would be appreciated. :) +The syntax is documented in the [jq manual]. + +[jq manual]: https://jqlang.github.io/jq/manual/v1.6/ + +Access a field: + + $ echo '{"a": 1, "b": 2}' | jaq '.a' + 1 + +Add values: + + $ echo '{"a": 1, "b": 2}' | jaq 'add' + 3 + +Construct an array from an object in two ways and show that they are equal: + + $ echo '{"a": 1, "b": 2}' | jaq '[.a, .b] == [.[]]' + true + +Apply a filter to all elements of an array and filter the results: + + $ echo '[0, 1, 2, 3]' | jaq 'map(.*2) | [.[] | select(. < 5)]' + [0, 2, 4] + +Read (slurp) input values into an array and get the average of its elements: + + $ echo '1 2 3 4' | jaq -s 'add / length' + 2.5 + +Repeatedly apply a filter to itself and output the intermediate results: + + $ echo '0' | jaq '[recurse(.+1; . < 3)]' + [0, 1, 2] + +Lazily fold over inputs and output intermediate results: + + $ seq 1000 | jaq -n 'foreach inputs as $x (0; . + $x)' + 1 3 6 10 15 [...] + + + +# Performance + +The following evaluation consists of several benchmarks that +allow comparing the performance of jaq, jq, and [gojq]. +The `empty` benchmark runs `n` times the filter `empty` with null input, +serving to measure the startup time. +The `bf-fib` benchmark runs a Brainfuck interpreter written in jq, +interpreting a Brainfuck script that produces `n` Fibonacci numbers. +The other benchmarks evaluate various filters with `n` as input; +see [`bench.sh`](bench.sh) for details. + +I generated the benchmark data with +`bench.sh target/release/jaq jq-1.8.1 gojq-0.12.17 | tee bench.json` +on a Linux system with an AMD Ryzen 5 5500U.[^binaries] +I then processed the results with a "one-liner" (stretching the term and the line a bit): + + jq -rs '.[] | "|`\(.name)`|\(.n)|" + ([.time[] | min | (.*1000|round)? // "N/A"] | min as $total_min | map(if . == $total_min then "**\(.)**" else "\(.)" end) | join("|"))' bench.json + +(Of course, you can also use jaq here instead of jq.) +Finally, I concatenated the table header with the output and piped it through `pandoc -t gfm`. + +[^binaries]: The binaries for jq-1.8.1 and gojq-0.12.17 were retrieved from their GitHub release pages. + +Table: Evaluation results in milliseconds ("N/A" if error or more than 10 seconds). + +| Benchmark | n | jaq-2.3 | jq-1.8.1 | gojq-0.12.17 | +|-----------------|--------:|--------:|---------:|-------------:| +| `empty` | 512 | 330 | 440 | **290** | +| `bf-fib` | 13 | **430** | 1110 | 540 | +| `defs` | 100000 | **60** | N/A | 1000 | +| `upto` | 8192 | **0** | 470 | 450 | +| `reduce-update` | 16384 | **10** | 490 | 1200 | +| `reverse` | 1048576 | **30** | 500 | 270 | +| `sort` | 1048576 | **100** | 450 | 540 | +| `group-by` | 1048576 | **340** | 1750 | 1540 | +| `min-max` | 1048576 | 190 | **170** | 260 | +| `add` | 1048576 | **440** | 570 | 1150 | +| `kv` | 131072 | **100** | 140 | 270 | +| `kv-update` | 131072 | **110** | 480 | 480 | +| `kv-entries` | 131072 | **520** | 1050 | 800 | +| `ex-implode` | 1048576 | **470** | 1010 | 590 | +| `reduce` | 1048576 | **720** | 850 | N/A | +| `try-catch` | 1048576 | **170** | 220 | 370 | +| `repeat` | 1048576 | **140** | 690 | 530 | +| `from` | 1048576 | **280** | 800 | 550 | +| `last` | 1048576 | **40** | 160 | 110 | +| `pyramid` | 524288 | 310 | **270** | 480 | +| `tree-contains` | 23 | **60** | 590 | 220 | +| `tree-flatten` | 17 | 750 | 340 | **0** | +| `tree-update` | 17 | **470** | 970 | 1300 | +| `tree-paths` | 17 | **130** | 250 | 770 | +| `to-fromjson` | 65536 | **40** | 370 | 100 | +| `ack` | 7 | **510** | 540 | 1090 | +| `range-prop` | 128 | 350 | 270 | **230** | +| `cumsum` | 1048576 | **250** | 260 | 460 | +| `cumsum-xy` | 1048576 | 400 | **350** | 680 | + +The results show that +jaq-2.3 is fastest on 23 benchmarks, whereas +jq-1.8.1 is fastest on 3 benchmark and +gojq-0.12.17 is fastest on 3 benchmarks. +gojq is much faster on `tree-flatten` because it implements the filter `flatten` natively instead of by definition. + +[gojq]: https://github.com/itchyny/gojq + + +# Security + +jaq's core has been audited by +[Radically Open Security](https://www.radicallyopensecurity.com/) +as part of an [NLnet](https://nlnet.nl/) grant --- +thanks to both organisations for their support! +The [security audit](https://github.com/01mf02/jaq/releases/download/v2.2.0/jaq.penetration.test.report.2025.1.0.pdf) found +one low severity issue and three issues that are likely not exploitable at all. +As a result of this security audit, all issues were addressed and +several fuzzing targets for jaq were added at `jaq-core/fuzz`. +Before that, jaq's JSON parser [hifijson](https://github.com/01mf02/hifijson/) +already disposed of a fuzzing target. +Finally, jaq disposes of a carefully crafted test suite of more than 500 tests +that is checked at every commit. + + +# Features + +Here is an overview that summarises: + +* [x] features already implemented, and +* [ ] features not yet implemented. + +[Contributions to extend jaq are highly welcome.](#contributing) + + +## Basics + +- [x] Identity (`.`) +- [x] Recursion (`..`) +- [x] Basic data types (null, boolean, number, string, array, object) +- [x] if-then-else (`if .a < .b then .a else .b end`) +- [x] Folding (`reduce .[] as $x (0; . + $x)`, `foreach .[] as $x (0; . + $x; . + .)`) +- [x] Error handling (`try ... catch ...`) +- [x] Breaking (`label $x | f | ., break $x`) +- [x] String interpolation (`"The successor of \(.) is \(.+1)."`) +- [x] Format strings (`@json`, `@text`, `@csv`, `@tsv`, `@html`, `@sh`, `@base64`, `@base64d`, `@uri`, `@urid`) + + +## Paths + +- [x] Indexing of arrays/objects (`.[0]`, `.a`, `.["a"]`) +- [x] Iterating over arrays/objects (`.[]`) +- [x] Optional indexing/iteration (`.a?`, `.[]?`) +- [x] Array slices (`.[3:7]`, `.[0:-1]`) +- [x] String slices + + +## Operators + +- [x] Composition (`|`) +- [x] Variable binding (`. as $x | $x`) +- [x] Pattern binding (`. as {a: [$x, {("b", "c"): $y, $z}]} | $x, $y, $z`) +- [x] Concatenation (`,`) +- [x] Plain assignment (`=`) +- [x] Update assignment (`|=`) +- [x] Arithmetic update assignment (`+=`, `-=`, ...) +- [x] Alternation (`//`) +- [x] Logic (`or`, `and`) +- [x] Equality and comparison (`.a == .b`, `.a < .b`) +- [x] Arithmetic (`+`, `-`, `*`, `/`, `%`) +- [x] Negation (`-`) +- [x] Error suppression (`?`) + + +## Definitions + +- [x] Basic definitions (`def map(f): [.[] | f];`) +- [x] Recursive definitions (`def r: r; r`) + + +## Core filters + +- [x] Empty (`empty`) +- [x] Errors (`error`) +- [x] Input (`inputs`) +- [x] Length (`length`, `utf8bytelength`) +- [x] Rounding (`floor`, `round`, `ceil`) +- [x] String <-> JSON (`fromjson`, `tojson`) +- [x] String <-> integers (`explode`, `implode`) +- [x] String normalisation (`ascii_downcase`, `ascii_upcase`) +- [x] String prefix/postfix (`startswith`, `endswith`, `ltrimstr`, `rtrimstr`) +- [x] String whitespace trimming (`trim`, `ltrim`, `rtrim`) +- [x] String splitting (`split("foo")`) +- [x] Array filters (`reverse`, `sort`, `sort_by(-.)`, `group_by`, `min_by`, `max_by`, `bsearch`) +- [x] Stream consumers (`first`, `last`, `range`, `fold`) +- [x] Stream generators (`range`, `recurse`) +- [x] Time (`now`, `fromdateiso8601`, `todateiso8601`) +- [x] More numeric filters (`sqrt`, `sin`, `log`, `pow`, ...) ([list of numeric filters](#numeric-filters)) +- [x] More time filters (`strptime`, `strftime`, `strflocaltime`, `mktime`, `gmtime`, and `localtime`) + +## Standard filters + +These filters are defined via more basic filters. +Their definitions are at [`std.jq`](jaq-std/src/std.jq). + +- [x] Undefined (`null`) +- [x] Booleans (`true`, `false`, `not`) +- [x] Special numbers (`nan`, `infinite`, `isnan`, `isinfinite`, `isfinite`, `isnormal`) +- [x] Type (`type`) +- [x] Filtering (`select(. >= 0)`) +- [x] Selection (`values`, `nulls`, `booleans`, `numbers`, `strings`, `arrays`, `objects`, `iterables`, `scalars`) +- [x] Conversion (`tostring`, `tonumber`) +- [x] Iterable filters (`map(.+1)`, `map_values(.+1)`, `add`, `join("a")`) +- [x] Array filters (`transpose`, `first`, `last`, `nth(10)`, `flatten`, `min`, `max`) +- [x] Object-array conversion (`to_entries`, `from_entries`, `with_entries`) +- [x] Universal/existential (`all`, `any`) +- [x] Recursion (`walk`) +- [x] I/O (`input`) +- [x] Regular expressions (`test`, `scan`, `match`, `capture`, `splits`, `sub`, `gsub`) +- [x] Time (`fromdate`, `todate`) + +## Numeric filters + +jaq imports many filters from [libm](https://crates.io/crates/libm) +and follows their type signature. + +

Full list of numeric filters defined in jaq + +Zero-argument filters: + +- [x] `acos` +- [x] `acosh` +- [x] `asin` +- [x] `asinh` +- [x] `atan` +- [x] `atanh` +- [x] `cbrt` +- [x] `cos` +- [x] `cosh` +- [x] `erf` +- [x] `erfc` +- [x] `exp` +- [x] `exp10` +- [x] `exp2` +- [x] `expm1` +- [x] `fabs` +- [x] `frexp`, which returns pairs of (float, integer). +- [x] `gamma` +- [x] `ilogb`, which returns integers. +- [x] `j0` +- [x] `j1` +- [x] `lgamma` +- [x] `log` +- [x] `log10` +- [x] `log1p` +- [x] `log2` +- [x] `logb` +- [x] `modf`, which returns pairs of (float, float). +- [x] `nearbyint` +- [x] `pow10` +- [x] `rint` +- [x] `significand` +- [x] `sin` +- [x] `sinh` +- [x] `sqrt` +- [x] `tan` +- [x] `tanh` +- [x] `tgamma` +- [x] `trunc` +- [x] `y0` +- [x] `y1` + +Two-argument filters that ignore `.`: + +- [x] `atan2` +- [x] `copysign` +- [x] `drem` +- [x] `fdim` +- [x] `fmax` +- [x] `fmin` +- [x] `fmod` +- [x] `hypot` +- [x] `jn`, which takes an integer as first argument. +- [x] `ldexp`, which takes an integer as second argument. +- [x] `nextafter` +- [x] `nexttoward` +- [x] `pow` +- [x] `remainder` +- [x] `scalb` +- [x] `scalbln`, which takes as integer as second argument. +- [x] `yn`, which takes an integer as first argument. + +Three-argument filters that ignore `.`: + +- [x] `fma` + +
+ +## Modules + +- [x] `include "path";` +- [x] `import "path" as mod;` +- [x] `import "path" as $data;` + +## Advanced features + +jaq currently does *not* aim to support several features of jq, such as: + +- SQL-style operators +- Streaming + + + +# Differences between jq and jaq + + +## Numbers + +jq uses 64-bit floating-point numbers (floats) for any number. +By contrast, jaq interprets +numbers such as 0 or -42 as machine-sized integers and +numbers such as 0.0 or 3e8 as 64-bit floats. +Many operations in jaq, such as array indexing, +check whether the passed numbers are indeed integer. +The motivation behind this is to avoid +rounding errors that may silently lead to wrong results. +For example: + + $ jq -n '[0, 1, 2] | .[1.0000000000000001]' + 1 + $ jaq -n '[0, 1, 2] | .[1.0000000000000001]' + Error: cannot use 1.0 as integer + $ jaq -n '[0, 1, 2] | .[1]' + 1 + +The rules of jaq are: + +* The sum, difference, product, and remainder of two integers is integer. +* Any other operation between two numbers yields a float. + +Examples: + + $ jaq -n '1 + 2' + 3 + $ jaq -n '10 / 2' + 5.0 + $ jaq -n '1.0 + 2' + 3.0 + +You can convert an integer to a floating-point number e.g. +by adding 0.0, by multiplying with 1.0, or by dividing with 1. +You can convert a floating-point number to an integer by +`round`, `floor`, or `ceil`: + + $ jaq -n '1.2 | [floor, round, ceil]' + [1, 1, 2] + +### NaN and infinity + +In jq, division by 0 yields an error, whereas +in jaq, `n / 0` yields `nan` if `n == 0`, `infinite` if `n > 0`, and `-infinite` if `n < 0`. +jaq's behaviour is closer to the IEEE standard for floating-point arithmetic (IEEE 754). + + +## Assignments + +Like jq, jaq allows for assignments of the form `p |= f`. +However, jaq interprets these assignments differently. +Fortunately, in most cases, the result is the same. + +In jq, an assignment `p |= f` first constructs paths to all values that match `p`. +*Only then*, it applies the filter `f` to these values. + +In jaq, an assignment `p |= f` applies `f` *immediately* to any value matching `p`. +Unlike in jq, assignment does not explicitly construct paths. + +jaq's implementation of assignment likely yields higher performance, +because it does not construct paths. +Furthermore, this allows jaq to use multiple outputs of the right-hand side, whereas +jq uses only the first. +For example, `0 | (., .) |= (., .+1)` yields `0 1 1 2` in jaq, +whereas it yields only `0` in jq. +However, `{a: 1} | .a |= (2, 3)` yields `{"a": 2}` in both jaq and jq, +because an object can only associate a single value with any given key, +so we cannot use multiple outputs in a meaningful way here. + +Because jaq does not construct paths, +it does not allow some filters on the left-hand side of assignments, +for example `first`, `last`, `limit`: +For example, `[1, 2, 3] | first(.[]) |= .-1` +yields `[0, 2, 3]` in jq, but is invalid in jaq. +Similarly, `[1, 2, 3] | limit(2; .[]) |= .-1` +yields `[0, 1, 3]` in jq, but is invalid in jaq. +(Inconsequentially, jq also does not allow for `last`.) + + +## Folding + +jq and jaq provide filters +`reduce xs as $x (init; update)`, +`foreach xs as $x (init; update)`, and +`foreach xs as $x (init; update; project)`, where +`foreach xs as $x (init; update)` is equivalent to +`foreach xs as $x (init; update; .)`. + +In jaq, the output of these filters is defined very simply: +Assuming that `xs` evaluates to `x0`, `x1`, ..., `xn`, +`reduce xs as $x (init; update)` evaluates to + +~~~ +init +| x0 as $x | update +| ... +| xn as $x | update +~~~ + +and `foreach xs as $x (init; update; project)` evaluates to + +~~~ text +init | +( x0 as $x | update | project, +( ... +( xn as $x | update | project, +( empty )...) +~~~ + +The interpretation of `reduce`/`foreach` in jaq has the following advantages over jq: + +* It deals very naturally with filters that yield multiple outputs. + In contrast, jq discriminates outputs of `f`, + because it recurses only on the last of them, + although it outputs all of them. +
Example + + `foreach (5, 10) as $x (1; .+$x, -.)` yields + `6, -1, 9, 1` in jq, whereas it yields + `6, 16, -6, -1, 9, 1` in jaq. + We can see that both jq and jaq yield the values `6` and `-1` + resulting from the first iteration (where `$x` is 5), namely + `1 | 5 as $x | (.+$x, -.)`. + However, jq performs the second iteration (where `$x` is 10) + *only on the last value* returned from the first iteration, namely `-1`, + yielding the values `9` and `1` resulting from + `-1 | 10 as $x | (.+$x, -.)`. + jaq yields these values too, but it also performs the second iteration + on all other values returned from the first iteration, namely `6`, + yielding the values `16` and `-6` that result from + ` 6 | 10 as $x | (.+$x, -.)`. + +
+* It makes the implementation of `reduce` and `foreach` + special cases of the same code, reducing the potential for bugs. + + +## Miscellaneous + +* Slurping: When files are slurped in (via the `-s` / `--slurp` option), + jq combines the inputs of all files into one single array, whereas + jaq yields an array for every file. + This is motivated by the `-i` / `--in-place` option, + which could not work with the behaviour implemented by jq. + The behaviour of jq can be approximated in jaq; + for example, to achieve the output of + `jq -s . a b`, you may use + `jaq -s . <(cat a b)`. +* Cartesian products: + In jq, `[(1,2) * (3,4)]` yields `[3, 6, 4, 8]`, whereas + `[{a: (1,2), b: (3,4)} | .a * .b]` yields `[3, 4, 6, 8]`. + jaq yields `[3, 4, 6, 8]` in both cases. +* Indexing `null`: + In jq, when given `null` input, `.["a"]` and `.[0]` yield `null`, but `.[]` yields an error. + jaq yields an error in all cases to prevent accidental indexing of `null` values. + To obtain the same behaviour in jq and jaq, you can use + `.["a"]? // null` or `.[0]? // null` instead. +* List updating: + In jq, `[0, 1] | .[3] = 3` yields `[0, 1, null, 3]`; that is, + jq fills up the list with `null`s if we update beyond its size. + In contrast, jaq fails with an out-of-bounds error in such a case. +* Input reading: + When there is no more input value left, + in jq, `input` yields an error, whereas in jaq, it yields no output value. +* Joining: + When giving an array `[x1, ..., xn]` to `join($sep)`, jaq returns + `""` if the array is empty, otherwise `"\(x1)" + $sep + ... + $sep + "\(xn)"`; + that is, it concatenates the string representations of the array values interspersed with `$sep`. + Unlike jq, jaq does not map `null` values in the array to `""`, + nor does it reject array or object values in the array. + + + +# Contributing + +Contributions to jaq are welcome. +Please make sure that after your change, `cargo test` runs successfully. + + + +# Acknowledgements + +[This project](https://nlnet.nl/project/jaq/) was funded through the +NGI0 Entrust Fund, a fund established by +NLnet with financial support from the +European Commission's Next Generation Internet +programme, under the aegis of DG Communications Networks, Content and Technology under grant agreement No 101069594. + +jaq has also profited from: + +* [serde_json] to read and [colored_json] to output JSON, +* [chumsky] to parse and [ariadne] to pretty-print parse errors, +* [mimalloc] to boost the performance of memory allocation, and +* the Rust standard library, in particular its awesome [Iterator], + which builds the rock-solid base of jaq's filter execution + +[serde_json]: https://docs.rs/serde_json/ +[colored_json]: https://docs.rs/colored_json/ +[chumsky]: https://docs.rs/chumsky/ +[ariadne]: https://docs.rs/ariadne/ +[mimalloc]: https://docs.rs/mimalloc/ +[Iterator]: https://doc.rust-lang.org/std/iter/trait.Iterator.html diff --git a/userland/jaq/src/cli.rs b/userland/jaq/src/cli.rs new file mode 100644 index 0000000..7c627ec --- /dev/null +++ b/userland/jaq/src/cli.rs @@ -0,0 +1,233 @@ +//! Command-line argument parsing +use core::fmt; +use std::env::ArgsOs; +use std::ffi::OsString; +use std::path::PathBuf; + +#[derive(Debug, Default)] +pub struct Cli { + // Input options + pub null_input: bool, + /// When the option `--slurp` is used additionally, + /// then the whole input is read into a single string. + pub raw_input: bool, + /// When input is read from files, + /// jaq yields an array for each file, whereas + /// jq produces only a single array. + pub slurp: bool, + + // Output options + pub compact_output: bool, + pub raw_output: bool, + /// This flag enables `--raw-output`. + pub join_output: bool, + pub in_place: bool, + pub sort_keys: bool, + pub color_output: bool, + pub monochrome_output: bool, + pub tab: bool, + pub indent: usize, + + // Compilation options + pub from_file: bool, + /// If this option is given multiple times, all given directories are searched. + pub library_path: Vec, + + // Key-value options + pub arg: Vec<(String, String)>, + pub argjson: Vec<(String, String)>, + pub slurpfile: Vec<(String, OsString)>, + pub rawfile: Vec<(String, OsString)>, + + // Positional arguments + /// If this argument is not given, it is assumed to be `.`, the identity filter. + pub filter: Option, + pub files: Vec, + pub args: Vec, + //pub jsonargs: Vec, + pub run_tests: Option>, + /// If there is some last output value `v`, + /// then the exit status code is + /// 1 if `v < true` (that is, if `v` is `false` or `null`) and + /// 0 otherwise. + /// If there is no output value, then the exit status code is 4. + /// + /// If any error occurs, then this option has no effect. + pub exit_status: bool, + pub version: bool, + pub help: bool, +} + +#[derive(Debug)] +pub enum Filter { + Inline(String), + FromFile(PathBuf), +} + +impl Cli { + fn positional(&mut self, mode: &Mode, arg: OsString) -> Result<(), Error> { + if self.filter.is_none() { + self.filter = Some(if self.from_file { + Filter::FromFile(arg.into()) + } else { + Filter::Inline(arg.into_string()?) + }) + } else { + match mode { + Mode::Files => self.files.push(arg.into()), + Mode::Args => self.args.push(arg.into_string()?), + //Mode::JsonArgs => self.jsonargs.push(arg.into_string()?), + } + } + Ok(()) + } + + fn long(&mut self, mode: &mut Mode, arg: &str, args: &mut ArgsOs) -> Result<(), Error> { + let int = |s: OsString| s.into_string().ok()?.parse().ok(); + match arg { + // handle all arguments after "--" + "" => args.try_for_each(|arg| self.positional(mode, arg))?, + + "null-input" => self.short('n', args)?, + "raw-input" => self.short('R', args)?, + "slurp" => self.short('s', args)?, + + "compact-output" => self.short('c', args)?, + "raw-output" => self.short('r', args)?, + "join-output" => self.short('j', args)?, + "in-place" => self.short('i', args)?, + "sort-keys" => self.short('S', args)?, + "color-output" => self.short('C', args)?, + "monochrome-output" => self.short('M', args)?, + "tab" => self.tab = true, + "indent" => self.indent = args.next().and_then(int).ok_or(Error::Int("--indent"))?, + "from-file" => self.short('f', args)?, + "library-path" => self.short('L', args)?, + "arg" => { + let (name, value) = parse_key_val("--arg", args)?; + self.arg.push((name, value.into_string()?)); + } + "argjson" => { + let (name, value) = parse_key_val("--argjson", args)?; + self.argjson.push((name, value.into_string()?)); + } + "slurpfile" => self.slurpfile.push(parse_key_val("--slurpfile", args)?), + "rawfile" => self.rawfile.push(parse_key_val("--rawfile", args)?), + + "args" => *mode = Mode::Args, + //"jsonargs" => *mode = Mode::JsonArgs, + "run-tests" => self.run_tests = Some(args.map(PathBuf::from).collect()), + "exit-status" => self.short('e', args)?, + "version" => self.short('V', args)?, + "help" => self.short('h', args)?, + + arg => Err(Error::Flag(format!("--{arg}")))?, + } + Ok(()) + } + + fn short(&mut self, arg: char, args: &mut ArgsOs) -> Result<(), Error> { + match arg { + 'n' => self.null_input = true, + 'R' => self.raw_input = true, + 's' => self.slurp = true, + + 'c' => self.compact_output = true, + 'r' => self.raw_output = true, + 'j' => self.join_output = true, + 'i' => self.in_place = true, + 'S' => self.sort_keys = true, + 'C' => self.color_output = true, + 'M' => self.monochrome_output = true, + + 'f' => self.from_file = true, + 'L' => self + .library_path + .push(args.next().ok_or(Error::Path("-L"))?.into()), + 'e' => self.exit_status = true, + 'V' => self.version = true, + 'h' => self.help = true, + arg => Err(Error::Flag(format!("-{arg}")))?, + } + Ok(()) + } + + pub fn parse() -> Result { + let mut cli = Self { + indent: 2, + ..Self::default() + }; + let mut mode = Mode::Files; + let mut args = std::env::args_os(); + args.next(); + while let Some(arg) = args.next() { + match arg.to_str() { + // we've got a valid UTF-8 argument here + Some(s) => match s.strip_prefix("--") { + Some(rest) => cli.long(&mut mode, rest, &mut args)?, + None => match s.strip_prefix("-") { + Some(rest) => rest.chars().try_for_each(|c| cli.short(c, &mut args))?, + None => cli.positional(&mode, arg)?, + }, + }, + // we've got invalid UTF-8, so it is no valid flag + // note that we do not check here whether arg starts with `-`, + // because this seems to be quite difficult to do in a portable way + None => cli.positional(&mode, arg)?, + } + } + Ok(cli) + } + + pub fn color_if(&self, f: impl Fn() -> bool) -> bool { + if self.monochrome_output { + false + } else if self.color_output { + true + } else { + f() + } + } +} + +#[derive(Debug)] +pub enum Error { + Flag(String), + Utf8(OsString), + KeyValue(&'static str), + Int(&'static str), + Path(&'static str), +} + +impl fmt::Display for Error { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match self { + Self::Flag(s) => write!(f, "unknown flag: {s}"), + Self::Utf8(s) => write!(f, "invalid UTF-8: {s:?}"), + Self::KeyValue(o) => write!(f, "{o} expects a key and a value"), + Self::Int(o) => write!(f, "{o} expects an integer"), + Self::Path(o) => write!(f, "{o} expects a path"), + } + } +} + +/// Conversion of errors from [`OsString::into_string`]. +impl From for Error { + fn from(e: OsString) -> Self { + Self::Utf8(e) + } +} + +fn parse_key_val(arg: &'static str, args: &mut ArgsOs) -> Result<(String, OsString), Error> { + let err = || Error::KeyValue(arg); + let key = args.next().ok_or_else(err)?.into_string()?; + let val = args.next().ok_or_else(err)?; + Ok((key, val)) +} + +/// Interpretation of positional arguments. +enum Mode { + Args, + //JsonArgs, + Files, +} diff --git a/userland/jaq/src/filter.rs b/userland/jaq/src/filter.rs new file mode 100644 index 0000000..49d3e41 --- /dev/null +++ b/userland/jaq/src/filter.rs @@ -0,0 +1,240 @@ +//! Filter parsing, compilation, and execution. +use crate::{read, Cli, Error, Val}; +#[cfg(not(target_os = "wasi"))] +use crate::repl; +use core::fmt::{self, Display, Formatter}; +use jaq_core::{compile, load, Ctx, Native, RcIter, ValT}; +use std::{io, path::PathBuf}; + +pub type Filter = jaq_core::Filter>; + +pub fn parse_compile( + path: &PathBuf, + code: &str, + vars: &[String], + paths: &[PathBuf], +) -> Result<(Vec, Filter), Vec> { + use compile::Compiler; + use load::{import, Arena, File, Loader}; + + let default = ["~/.jq", "$ORIGIN/../lib/jq", "$ORIGIN/../lib"].map(|x| x.into()); + let paths = if paths.is_empty() { &default } else { paths }; + + let vars: Vec<_> = vars.iter().map(|v| format!("${v}")).collect(); + let arena = Arena::default(); + let defs = jaq_std::defs().chain(jaq_json::defs()); + let loader = Loader::new(defs).with_std_read(paths); + //let loader = Loader::new([]).with_std_read(paths); + let path = path.into(); + let modules = loader + .load(&arena, File { path, code }) + .map_err(load_errors)?; + + let mut vals = Vec::new(); + import(&modules, |p| { + let path = p.find(paths, "json")?; + vals.push(read::json_array(path).map_err(|e| e.to_string())?); + Ok(()) + }) + .map_err(load_errors)?; + + let funs = jaq_std::funs().chain(jaq_json::funs()); + #[cfg(not(target_os = "wasi"))] + let funs = funs.chain([repl::fun()]); + let compiler = Compiler::default() + .with_funs(funs) + .with_global_vars(vars.iter().map(|v| &**v)); + let filter = compiler.compile(modules).map_err(compile_errors)?; + Ok((vals, filter)) +} + +/// Run a filter with given input values and run `f` for every value output. +/// +/// This function cannot return an `Iterator` because it creates an `RcIter`. +/// This is most unfortunate. We should think about how to simplify this ... +pub(crate) fn run( + cli: &Cli, + filter: &Filter, + vars: Vec, + iter: impl Iterator>, + mut f: impl FnMut(Val) -> io::Result<()>, +) -> Result, Error> { + let mut last = None; + let iter = iter.map(|r| r.map_err(|e| e.to_string())); + + let iter = Box::new(iter) as Box>; + let null = Box::new(core::iter::once(Ok(Val::Null))) as Box>; + + let iter = RcIter::new(iter); + let null = RcIter::new(null); + + let ctx = Ctx::new(vars, &iter); + + for item in if cli.null_input { &null } else { &iter } { + let input = item.map_err(Error::Parse)?; + //println!("Got {:?}", input); + for output in filter.run((ctx.clone(), input)) { + let output = output.map_err(Error::Jaq)?; + last = Some(output.as_bool()); + f(output)?; + } + } + Ok(last) +} + +#[derive(Debug)] +pub struct FileReports(load::File, Vec); + +impl Display for FileReports { + fn fmt(&self, f: &mut Formatter) -> fmt::Result { + let Self(file, reports) = self; + let idx = codesnake::LineIndex::new(&file.code); + reports.iter().try_for_each(|e| { + writeln!(f, "Error: {}", e.message)?; + let block = e.to_block(&idx); + writeln!(f, "{}[{}]", block.prologue(), file.path.display())?; + writeln!(f, "{}{}", block, block.epilogue()) + }) + } +} + +fn load_errors(errs: load::Errors<&str, PathBuf>) -> Vec { + use load::Error; + + let errs = errs.into_iter().map(|(file, err)| { + let code = file.code; + let err = match err { + Error::Io(errs) => errs.into_iter().map(|e| report_io(code, e)).collect(), + Error::Lex(errs) => errs.into_iter().map(|e| report_lex(code, e)).collect(), + Error::Parse(errs) => errs.into_iter().map(|e| report_parse(code, e)).collect(), + }; + FileReports(file.map_code(|s| s.into()), err) + }); + errs.collect() +} + +fn compile_errors(errs: compile::Errors<&str, PathBuf>) -> Vec { + let errs = errs.into_iter().map(|(file, errs)| { + let code = file.code; + let errs = errs.into_iter().map(|e| report_compile(code, e)).collect(); + FileReports(file.map_code(|s| s.into()), errs) + }); + errs.collect() +} + +type StringColors = Vec<(String, Option)>; + +#[derive(Debug)] +struct Report { + message: String, + labels: Vec<(core::ops::Range, StringColors, Color)>, +} + +#[derive(Clone, Debug)] +enum Color { + Yellow, + Red, +} + +impl Color { + fn apply(&self, d: impl Display) -> String { + use yansi::{Color, Paint}; + let color = match self { + Self::Yellow => Color::Yellow, + Self::Red => Color::Red, + }; + d.fg(color).to_string() + } +} + +fn report_io(code: &str, (path, error): (&str, String)) -> Report { + let path_range = load::span(code, path); + Report { + message: format!("could not load file {}: {}", path, error), + labels: [(path_range, [(error, None)].into(), Color::Red)].into(), + } +} + +fn report_lex(code: &str, (expected, found): load::lex::Error<&str>) -> Report { + // truncate found string to its first character + let found = &found[..found.char_indices().nth(1).map_or(found.len(), |(i, _)| i)]; + + let found_range = load::span(code, found); + let found = match found { + "" => [("unexpected end of input".to_string(), None)].into(), + c => [("unexpected character ", None), (c, Some(Color::Red))] + .map(|(s, c)| (s.into(), c)) + .into(), + }; + let label = (found_range, found, Color::Red); + + let labels = match expected { + load::lex::Expect::Delim(open) => { + let text = [("unclosed delimiter ", None), (open, Some(Color::Yellow))] + .map(|(s, c)| (s.into(), c)); + Vec::from([(load::span(code, open), text.into(), Color::Yellow), label]) + } + _ => Vec::from([label]), + }; + + Report { + message: format!("expected {}", expected.as_str()), + labels, + } +} + +fn report_parse(code: &str, (expected, found): load::parse::Error<&str>) -> Report { + let found_range = load::span(code, found); + + let found = if found.is_empty() { + "unexpected end of input" + } else { + "unexpected token" + }; + let found = [(found.to_string(), None)].into(); + + Report { + message: format!("expected {}", expected.as_str()), + labels: Vec::from([(found_range, found, Color::Red)]), + } +} + +fn report_compile(code: &str, (found, undefined): compile::Error<&str>) -> Report { + use compile::Undefined::Filter; + let found_range = load::span(code, found); + let wnoa = |exp, got| format!("wrong number of arguments (expected {exp}, found {got})"); + let message = match (found, undefined) { + ("reduce", Filter(arity)) => wnoa("2", arity), + ("foreach", Filter(arity)) => wnoa("2 or 3", arity), + (_, undefined) => format!("undefined {}", undefined.as_str()), + }; + let found = [(message.clone(), None)].into(); + + Report { + message, + labels: Vec::from([(found_range, found, Color::Red)]), + } +} + +type CodeBlock = codesnake::Block, String>; + +impl Report { + fn to_block(&self, idx: &codesnake::LineIndex) -> CodeBlock { + use codesnake::{Block, CodeWidth, Label}; + let color_maybe = |(text, color): (_, Option)| match color { + None => text, + Some(color) => color.apply(text).to_string(), + }; + let labels = self.labels.iter().cloned().map(|(range, text, color)| { + let text = text.into_iter().map(color_maybe).collect::>(); + Label::new(range) + .with_text(text.join("")) + .with_style(move |s| color.apply(s).to_string()) + }); + Block::new(idx, labels).unwrap().map_code(|c| { + let c = c.replace('\t', " "); + let w = unicode_width::UnicodeWidthStr::width(&*c); + CodeWidth::new(c, core::cmp::max(w, 1)) + }) + } +} diff --git a/userland/jaq/src/help.txt b/userland/jaq/src/help.txt new file mode 100644 index 0000000..e75cd7e --- /dev/null +++ b/userland/jaq/src/help.txt @@ -0,0 +1,40 @@ +Just Another Query Tool + +Usage: jaq [OPTION]... [FILTER] [ARG]... + +Arguments: + [FILTER] Filter to execute + [ARG]... Positional arguments, by default used as input files + +Input options: + -n, --null-input Use null as single input value + -R, --raw-input Read lines of the input as sequence of strings + -s, --slurp Read (slurp) all input values into one array + +Output options: + -c, --compact-output Print JSON compactly, omitting whitespace + -r, --raw-output Write strings without escaping them with quotes + -j, --join-output Do not print a newline after each value + -i, --in-place Overwrite input file with its output + -S, --sort-keys Print objects sorted by their keys + -C, --color-output Always color output + -M, --monochrome-output Do not color output + --tab Use tabs for indentation rather than spaces + --indent Use N spaces for indentation [default: 2] + +Compilation options: + -f, --from-file Read filter from a file given by filter argument + -L, --library-path Search for modules and data in given directory + +Variable options: + --arg Set variable `$A` to string `V` + --argjson Set variable `$A` to JSON value `V` + --slurpfile Set variable `$A` to array containing the JSON values in file `F` + --rawfile Set variable `$A` to string containing the contents of file `F` + --args Collect remaining positional arguments into `$ARGS.positional` + +Remaining options: + --run-tests Run tests from a file + -e, --exit-status Use the last output value as exit status code + -V, --version Print version + -h, --help Print help diff --git a/userland/jaq/src/main.rs b/userland/jaq/src/main.rs new file mode 100644 index 0000000..0819ad6 --- /dev/null +++ b/userland/jaq/src/main.rs @@ -0,0 +1,276 @@ +mod cli; +mod filter; +mod read; +#[cfg(not(target_os = "wasi"))] +mod repl; +mod write; + +use cli::Cli; +use core::fmt::{self, Display, Formatter}; +use filter::{run, FileReports, Filter}; +use is_terminal::IsTerminal; +use jaq_core::{load, Ctx, RcIter}; +use jaq_json::Val; +use std::io::{self, BufRead, Write}; +use std::path::{Path, PathBuf}; +use std::process::{ExitCode, Termination}; +use write::{print, with_stdout}; + +#[cfg(feature = "mimalloc")] +#[global_allocator] +static GLOBAL: mimalloc::MiMalloc = mimalloc::MiMalloc; + +fn main() -> ExitCode { + use env_logger::Env; + env_logger::Builder::from_env(Env::default().filter_or("LOG", "jaq=debug")) + .format(|buf, record| match record.level() { + // format error messages (yielded by `stderr`) without newline + log::Level::Error => write!(buf, "{}", record.args()), + // format debug messages such as `["DEBUG:", [1, 2, 3]]` + level => writeln!(buf, "[\"{}:\", {}]", level, record.args()), + }) + .init(); + + let cli = match Cli::parse() { + Ok(cli) => cli, + Err(e) => { + eprintln!("Error: {e}"); + return ExitCode::from(2); + } + }; + + if cli.version { + println!("{} {}", env!("CARGO_PKG_NAME"), env!("CARGO_PKG_VERSION")); + return ExitCode::SUCCESS; + } else if cli.help { + println!("{}", include_str!("help.txt")); + return ExitCode::SUCCESS; + } + + let no_color = std::env::var("NO_COLOR").map_or(false, |v| !v.is_empty()); + let set_color = |on| { + if on { + yansi::enable(); + } else { + yansi::disable(); + } + }; + + set_color(!cli.in_place && cli.color_if(|| std::io::stdout().is_terminal() && !no_color)); + + match real_main(&cli) { + Ok(exit) => exit, + Err(e) => { + set_color(cli.color_if(|| std::io::stderr().is_terminal() && !no_color)); + eprint!("{e}"); + e.report() + } + } +} + +fn real_main(cli: &Cli) -> Result { + if let Some(test_files) = &cli.run_tests { + return Ok(match test_files.last() { + Some(file) => run_tests(io::BufReader::new(std::fs::File::open(file)?)), + None => run_tests(io::stdin().lock()), + }); + } + + let (vars, mut ctx): (Vec, Vec) = binds(cli)?.into_iter().unzip(); + + let (vals, filter) = match &cli.filter { + None => (Vec::new(), Filter::default()), + Some(filter) => { + let (path, code) = match filter { + cli::Filter::FromFile(path) => (path.into(), std::fs::read_to_string(path)?), + cli::Filter::Inline(filter) => ("".into(), filter.clone()), + }; + filter::parse_compile(&path, &code, &vars, &cli.library_path).map_err(Error::Report)? + } + }; + ctx.extend(vals); + //println!("Filter: {:?}", filter); + + let last = if cli.files.is_empty() { + let inputs = read::buffered(cli, io::stdin().lock()); + with_stdout(|out| run(cli, &filter, ctx, inputs, |v| print(out, cli, &v)))? + } else { + let mut last = None; + for file in &cli.files { + let path = Path::new(file); + let file = read::load_file(path) + .map_err(|e| Error::Io(Some(path.display().to_string()), e))?; + let inputs = read::slice(cli, &file); + if cli.in_place { + // create a temporary file where output is written to + let location = path.parent().unwrap(); + let mut tmp = tempfile::Builder::new() + .prefix("jaq") + .tempfile_in(location)?; + + last = run(cli, &filter, ctx.clone(), inputs, |output| { + print(tmp.as_file_mut(), cli, &output) + })?; + + // replace the input file with the temporary file + std::mem::drop(file); + let perms = std::fs::metadata(path)?.permissions(); + tmp.persist(path).map_err(Error::Persist)?; + std::fs::set_permissions(path, perms)?; + } else { + last = with_stdout(|out| { + run(cli, &filter, ctx.clone(), inputs, |v| print(out, cli, &v)) + })?; + } + } + last + }; + + if cli.exit_status { + last.map_or_else( + || Err(Error::NoOutput), + |b| b.then_some(ExitCode::SUCCESS).ok_or(Error::FalseOrNull), + ) + } else { + Ok(ExitCode::SUCCESS) + } +} + +fn binds(cli: &Cli) -> Result, Error> { + let arg = cli.arg.iter().map(|(k, s)| { + let s = s.to_owned(); + Ok((k.to_owned(), Val::Str(s.into()))) + }); + let argjson = cli.argjson.iter().map(|(k, s)| { + use hifijson::token::Lex; + let mut lexer = hifijson::SliceLexer::new(s.as_bytes()); + let err = |e| Error::Parse(format!("{e} (for value passed to `--argjson {k}`)")); + Ok((k.to_owned(), lexer.exactly_one(Val::parse).map_err(err)?)) + }); + let rawfile = cli.rawfile.iter().map(|(k, path)| { + let s = std::fs::read_to_string(path).map_err(|e| Error::Io(Some(format!("{path:?}")), e)); + Ok((k.to_owned(), Val::Str(s?.into()))) + }); + let slurpfile = cli.slurpfile.iter().map(|(k, path)| { + let a = read::json_array(path).map_err(|e| Error::Io(Some(format!("{path:?}")), e)); + Ok((k.to_owned(), a?)) + }); + + let positional = cli.args.iter().cloned().map(|s| Ok(Val::from(s))); + let positional = positional.collect::, Error>>()?; + + let var_val = arg.chain(rawfile).chain(slurpfile).chain(argjson); + let mut var_val = var_val.collect::, Error>>()?; + + var_val.push(("ARGS".to_string(), args(&positional, &var_val))); + let env = std::env::vars().map(|(k, v)| (k.into(), Val::from(v))); + var_val.push(("ENV".to_string(), Val::obj(env.collect()))); + + Ok(var_val) +} + +fn args(positional: &[Val], named: &[(String, Val)]) -> Val { + let key = |k: &str| k.to_string().into(); + let positional = positional.iter().cloned(); + let named = named.iter().map(|(var, val)| (key(var), val.clone())); + let obj = [ + (key("positional"), positional.collect()), + (key("named"), Val::obj(named.collect())), + ]; + Val::obj(obj.into_iter().collect()) +} + +#[derive(Debug)] +enum Error { + Io(Option, io::Error), + Report(Vec), + Parse(String), + Jaq(jaq_core::Error), + Persist(tempfile::PersistError), + FalseOrNull, + NoOutput, +} + +impl Display for Error { + fn fmt(&self, f: &mut Formatter) -> fmt::Result { + match self { + Self::FalseOrNull | Self::NoOutput => Ok(()), + Self::Io(prefix, e) => { + write!(f, "Error: ")?; + if let Some(p) = prefix { + write!(f, "{p}: ")?; + } + writeln!(f, "{e}") + } + Self::Persist(e) => { + writeln!(f, "Error: {e}") + } + Self::Report(reports) => reports.iter().try_for_each(|fr| write!(f, "{fr}")), + Self::Parse(e) => writeln!(f, "Error: failed to parse: {e}"), + Self::Jaq(e) => writeln!(f, "Error: {e}"), + } + } +} + +impl Termination for Error { + fn report(self) -> ExitCode { + ExitCode::from(match self { + Self::FalseOrNull => 1, + Self::Io(_, _) | Self::Persist(_) => 2, + Self::Report(_) => 3, + Self::NoOutput => 4, + Self::Parse(_) | Self::Jaq(_) => 5, + }) + } +} + +impl From for Error { + fn from(e: io::Error) -> Self { + Self::Io(None, e) + } +} + +fn run_test(test: load::test::Test) -> Result<(Val, Val), Error> { + let (ctx, filter) = + filter::parse_compile(&PathBuf::new(), &test.filter, &[], &[]).map_err(Error::Report)?; + + let inputs = RcIter::new(Box::new(core::iter::empty())); + let ctx = Ctx::new(ctx, &inputs); + + let json = |s: String| { + use hifijson::token::Lex; + hifijson::SliceLexer::new(s.as_bytes()) + .exactly_one(Val::parse) + .map_err(read::invalid_data) + }; + let input = json(test.input)?; + let expect: Result = test.output.into_iter().map(json).collect(); + let obtain: Result = filter.run((ctx, input)).collect(); + Ok((expect?, obtain.map_err(Error::Jaq)?)) +} + +fn run_tests(read: impl BufRead) -> ExitCode { + let lines = read.lines().map(Result::unwrap); + let tests = load::test::Parser::new(lines); + + let (mut passed, mut total) = (0, 0); + for test in tests { + println!("Testing {}", test.filter); + match run_test(test) { + Err(e) => eprintln!("{e:?}"), + Ok((expect, obtain)) if expect != obtain => { + eprintln!("expected {expect}, obtained {obtain}",); + } + Ok(_) => passed += 1, + } + total += 1; + } + + println!("{passed} out of {total} tests passed"); + + if total > passed { + ExitCode::FAILURE + } else { + ExitCode::SUCCESS + } +} diff --git a/userland/jaq/src/read.rs b/userland/jaq/src/read.rs new file mode 100644 index 0000000..4bddd41 --- /dev/null +++ b/userland/jaq/src/read.rs @@ -0,0 +1,81 @@ +use crate::{Cli, Val}; +use std::io::{self, BufRead}; +use std::path::Path; + +/// Try to load file by memory mapping and fall back to regular loading if it fails. +pub fn load_file(path: impl AsRef) -> io::Result>> { + let file = std::fs::File::open(path.as_ref())?; + match unsafe { memmap2::Mmap::map(&file) } { + Ok(mmap) => Ok(Box::new(mmap)), + Err(_) => Ok(Box::new(std::fs::read(path)?)), + } +} + +pub fn invalid_data(e: impl std::error::Error + Send + Sync + 'static) -> std::io::Error { + io::Error::new(io::ErrorKind::InvalidData, e) +} + +fn json_slice(slice: &[u8]) -> impl Iterator> + '_ { + let mut lexer = hifijson::SliceLexer::new(slice); + core::iter::from_fn(move || { + use hifijson::token::Lex; + Some(Val::parse(lexer.ws_token()?, &mut lexer).map_err(invalid_data)) + }) +} + +fn json_read<'a>(read: impl BufRead + 'a) -> impl Iterator> + 'a { + let mut lexer = hifijson::IterLexer::new(read.bytes()); + core::iter::from_fn(move || { + use hifijson::token::Lex; + let v = Val::parse(lexer.ws_token()?, &mut lexer); + Some(v.map_err(|e| core::mem::take(&mut lexer.error).unwrap_or_else(|| invalid_data(e)))) + }) +} + +pub fn json_array(path: impl AsRef) -> io::Result { + json_slice(&load_file(path.as_ref())?).collect() +} + +pub fn buffered<'a, R>(cli: &Cli, read: R) -> Box> + 'a> +where + R: BufRead + 'a, +{ + if cli.raw_input { + Box::new(raw_input(cli.slurp, read).map(|r| r.map(Val::from))) + } else { + Box::new(collect_if(cli.slurp, json_read(read))) + } +} + +pub fn slice<'a>(cli: &Cli, slice: &'a [u8]) -> Box> + 'a> { + if cli.raw_input { + let read = io::BufReader::new(slice); + Box::new(raw_input(cli.slurp, read).map(|r| r.map(Val::from))) + } else { + Box::new(collect_if(cli.slurp, json_slice(slice))) + } +} + +fn raw_input<'a, R>(slurp: bool, mut read: R) -> impl Iterator> + 'a +where + R: BufRead + 'a, +{ + if slurp { + let mut buf = String::new(); + let s = read.read_to_string(&mut buf).map(|_| buf); + Box::new(std::iter::once(s)) + } else { + Box::new(read.lines()) as Box> + } +} + +fn collect_if<'a, T: FromIterator + 'a, E: 'a>( + slurp: bool, + iter: impl Iterator> + 'a, +) -> Box> + 'a> { + if slurp { + Box::new(core::iter::once(iter.collect())) + } else { + Box::new(iter) + } +} diff --git a/userland/jaq/src/repl.rs b/userland/jaq/src/repl.rs new file mode 100644 index 0000000..8e03db4 --- /dev/null +++ b/userland/jaq/src/repl.rs @@ -0,0 +1,55 @@ +use crate::{filter, run, write, Cli, Error, Val}; +use jaq_core::Native; +use jaq_std::Filter; +use rustyline::error::ReadlineError; +use rustyline::DefaultEditor; +use std::sync::atomic::{AtomicUsize, Ordering}; + +// counter that increases for each nested invocation of `repl` +static DEPTH: AtomicUsize = AtomicUsize::new(0); + +pub fn fun() -> Filter> { + jaq_std::run(("repl", jaq_std::v(0), |_, cv| { + let depth = DEPTH.fetch_add(1, Ordering::Relaxed); + repl_with(depth, |s| match eval(s, cv.1.clone()) { + Ok(()) => (), + Err(e) => eprint!("{e}"), + }) + .unwrap(); + DEPTH.fetch_sub(1, Ordering::Relaxed); + + Box::new(core::iter::empty()) + })) +} + +fn eval(code: String, input: Val) -> Result<(), Error> { + let (ctx, filter) = + filter::parse_compile(&"".into(), &code, &[], &[]).map_err(Error::Report)?; + let cli = &Cli::default(); + let inputs = core::iter::once(Ok(input)); + crate::with_stdout(|out| run(cli, &filter, ctx, inputs, |v| write::print(out, cli, &v)))?; + Ok(()) +} + +fn repl_with(depth: usize, f: impl Fn(String)) -> Result<(), ReadlineError> { + use rustyline::config::{Behavior, Config}; + use yansi::Paint; + let config = Config::builder() + .behavior(Behavior::PreferTerm) + .auto_add_history(true) + .build(); + let mut rl = DefaultEditor::with_config(config)?; + let history = dirs::cache_dir().map(|dir| dir.join("jaq-history")); + let _ = history.iter().try_for_each(|h| rl.load_history(h)); + let prompt = format!("{}{} ", str::repeat(" ", depth), '>'.bold()); + loop { + match rl.readline(&prompt) { + Ok(line) => f(line), + Err(ReadlineError::Interrupted) => (), + Err(ReadlineError::Eof) => break, + Err(err) => Err(err)?, + } + } + let _ = history.iter().try_for_each(|h| rl.append_history(h)); + Ok(()) +} diff --git a/userland/jaq/src/write.rs b/userland/jaq/src/write.rs new file mode 100644 index 0000000..29691c4 --- /dev/null +++ b/userland/jaq/src/write.rs @@ -0,0 +1,125 @@ +use crate::{Cli, Val}; +use core::fmt::{self, Display, Formatter}; +use is_terminal::IsTerminal; +use std::io::{self, Write}; + +struct FormatterFn(F); + +impl fmt::Result> Display for FormatterFn { + fn fmt(&self, f: &mut Formatter) -> fmt::Result { + self.0(f) + } +} + +struct PpOpts { + compact: bool, + indent: String, + sort_keys: bool, +} + +impl PpOpts { + fn indent(&self, f: &mut Formatter, level: usize) -> fmt::Result { + if !self.compact { + write!(f, "{}", self.indent.repeat(level))?; + } + Ok(()) + } + + fn newline(&self, f: &mut Formatter) -> fmt::Result { + if !self.compact { + writeln!(f)?; + } + Ok(()) + } +} + +fn fmt_seq(fmt: &mut Formatter, opts: &PpOpts, level: usize, xs: I, f: F) -> fmt::Result +where + I: IntoIterator, + F: Fn(&mut Formatter, T) -> fmt::Result, +{ + opts.newline(fmt)?; + let mut iter = xs.into_iter().peekable(); + while let Some(x) = iter.next() { + opts.indent(fmt, level + 1)?; + f(fmt, x)?; + if iter.peek().is_some() { + write!(fmt, ",")?; + } + opts.newline(fmt)?; + } + opts.indent(fmt, level) +} + +fn fmt_val(f: &mut Formatter, opts: &PpOpts, level: usize, v: &Val) -> fmt::Result { + use yansi::Paint; + match v { + Val::Null | Val::Bool(_) | Val::Int(_) | Val::Float(_) | Val::Num(_) => v.fmt(f), + Val::Str(_) => write!(f, "{}", v.green()), + Val::Arr(a) => { + '['.bold().fmt(f)?; + if !a.is_empty() { + fmt_seq(f, opts, level, &**a, |f, x| fmt_val(f, opts, level + 1, x))?; + } + ']'.bold().fmt(f) + } + Val::Obj(o) => { + '{'.bold().fmt(f)?; + let kv = |f: &mut Formatter, (k, val): (&std::rc::Rc, &Val)| { + write!(f, "{}:", Val::Str(k.clone()).bold())?; + if !opts.compact { + write!(f, " ")?; + } + fmt_val(f, opts, level + 1, val) + }; + if !o.is_empty() { + if opts.sort_keys { + let mut o: Vec<_> = o.iter().collect(); + o.sort_by_key(|(k, _v)| *k); + fmt_seq(f, opts, level, o, kv) + } else { + fmt_seq(f, opts, level, &**o, kv) + }? + } + '}'.bold().fmt(f) + } + } +} + +pub fn print(w: &mut (impl Write + ?Sized), cli: &Cli, val: &Val) -> io::Result<()> { + let f = |f: &mut Formatter| { + let opts = PpOpts { + compact: cli.compact_output, + indent: if cli.tab { + String::from("\t") + } else { + " ".repeat(cli.indent) + }, + sort_keys: cli.sort_keys, + }; + fmt_val(f, &opts, 0, val) + }; + + match val { + Val::Str(s) if cli.raw_output || cli.join_output => write!(w, "{s}")?, + _ => write!(w, "{}", FormatterFn(f))?, + }; + + if cli.join_output { + // when running `jaq -jn '"prompt> " | (., input)'`, + // this flush is necessary to make "prompt> " appear first + w.flush() + } else { + // this also flushes output, because stdout is line-buffered in Rust + writeln!(w) + } +} + +pub fn with_stdout(f: impl FnOnce(&mut dyn Write) -> T) -> T { + let stdout = io::stdout(); + if stdout.is_terminal() { + f(&mut stdout.lock()) + } else { + f(&mut io::BufWriter::new(stdout.lock())) + } +} diff --git a/userland/jaq/tests/a.jq b/userland/jaq/tests/a.jq new file mode 100644 index 0000000..3ebbf49 --- /dev/null +++ b/userland/jaq/tests/a.jq @@ -0,0 +1,7 @@ +include "b" {search: "."}; +import "c" as c {search: "mods"}; +import "data" as $data {search: "."}; +def a: b + c::c; +def d: 2; +def d: 3; +def data: $data; diff --git a/userland/jaq/tests/b.jq b/userland/jaq/tests/b.jq new file mode 100644 index 0000000..d95dd10 --- /dev/null +++ b/userland/jaq/tests/b.jq @@ -0,0 +1 @@ +def b: "b"; diff --git a/userland/jaq/tests/data.json b/userland/jaq/tests/data.json new file mode 100644 index 0000000..1191247 --- /dev/null +++ b/userland/jaq/tests/data.json @@ -0,0 +1,2 @@ +1 +2 diff --git a/userland/jaq/tests/golden.rs b/userland/jaq/tests/golden.rs new file mode 100644 index 0000000..fd58322 --- /dev/null +++ b/userland/jaq/tests/golden.rs @@ -0,0 +1,157 @@ +use std::{env, io, process, str}; + +fn golden_test(args: &[&str], input: &str, out_ex: &str) -> io::Result<()> { + let mut child = process::Command::new(env!("CARGO_BIN_EXE_jaq")) + .args(args) + .stdin(process::Stdio::piped()) + .stdout(process::Stdio::piped()) + .spawn()?; + + use io::Write; + child.stdin.take().unwrap().write_all(input.as_bytes())?; + let output = child.wait_with_output()?; + assert!(output.status.success()); + + let out_act = str::from_utf8(&output.stdout).expect("invalid UTF-8 in output"); + // remove '\r' from output for compatibility with Windows + let out_act = out_act.replace('\r', ""); + if out_ex.trim() != out_act.trim() { + println!("Expected output:\n{}\n---", out_ex); + println!("Actual output:\n{}\n---", out_act); + process::exit(2); + } + Ok(()) +} + +macro_rules! test { + ($name:ident, $args:expr, $input:expr, $output:expr) => { + #[test] + fn $name() -> io::Result<()> { + golden_test($args, $input, $output) + } + }; +} + +test!(no_args, &[], "[0, 1]", "[\n 0,\n 1\n]"); +test!(one, &["1"], "0", "1"); +test!(sparse, &["."], "[2,3]", "[\n 2,\n 3\n]"); + +test!( + arg, + &["--arg", "x", "y", "--arg", "a", "b", "$x + $a"], + "0", + "\"yb\"" +); + +test!( + argjson, + &[ + "--argjson", + "a", + "[1,2,3]", + "--argjson", + "b", + r#""abc""#, + "$a,$b" + ], + "0", + r#"[ + 1, + 2, + 3 +] +"abc""# +); + +test!( + args, + &["-c", "--args", "--arg", "x", "y", "$ARGS", "a", "--", "--test", "--"], + "0", + r#"{"positional":["a","--test","--"],"named":{"x":"y"}}"# +); + +test!( + join_output, + &["-j", "."], + r#"[] "foo" "bar" 1 2 {}"#, + "[]foobar12{}" +); + +test!( + raw_output, + &["-r", "."], + r#""foo" "bar" ["baz"]"#, + r#"foo +bar +[ + "baz" +]"# +); + +test!( + compact, + &["-c", "."], + r#"[2,3] +{"a":1, "b":["c" ]}"#, + r#"[2,3] +{"a":1,"b":["c"]}"# +); + +test!( + sort_keys, + &["-Sc", "."], + r#"{"b": 1, "a": 2}"#, + r#"{"a":2,"b":1}"# +); + +test!( + inputs, + &["-c", r#"{".": .}, {input: input}"#], + "0\n1\n2\n3", + r#"{".":0} +{"input":1} +{".":2} +{"input":3}"# +); + +test!( + null_input, + &["-nc", r#"{".": .}, {inputs: [inputs]}"#], + "0\n1\n2\n3", + r#"{".":null} +{"inputs":[0,1,2,3]}"# +); + +const ONE23: &str = "One\nTwo\nThree\n"; + +test!(raw_input_slurp, &["-Rs"], ONE23, r#""One\nTwo\nThree\n""#); + +test!( + raw_input, + &["-R"], + ONE23, + r#""One" +"Two" +"Three""# +); + +test!( + fmt_str, + &[], + r#""\u0000\u200b\r\t\n asdf""#, + r#""\u0000​\r\t\n asdf""# +); + +test!( + fmt_obj_key, + &["-c"], + r#"{"विश\u094dव": 1}"#, + r#"{"विशà¥à¤µ":1}"# +); + +test!( + mods, + &["-c", "-L", "tests", r#"include "a"; [a, data, d]"#], + "0", + r#"["bcddd",[1,2],3]"# +); diff --git a/userland/jaq/tests/mods/c.jq b/userland/jaq/tests/mods/c.jq new file mode 100644 index 0000000..9ee9bb2 --- /dev/null +++ b/userland/jaq/tests/mods/c.jq @@ -0,0 +1,4 @@ +import "d" as d1 {search: "."}; +import "d" as d2 {search: "../mods"}; +import "d" as d3 {search: ["foo", "."]}; +def c: "c" + d1::d + d2::d + d3::d; diff --git a/userland/jaq/tests/mods/d.jq b/userland/jaq/tests/mods/d.jq new file mode 100644 index 0000000..873d6f5 --- /dev/null +++ b/userland/jaq/tests/mods/d.jq @@ -0,0 +1,2 @@ +include "b" {search: ".."}; +def d: "d"; diff --git a/userland/ripgrep/.cargo-ok b/userland/ripgrep/.cargo-ok new file mode 100644 index 0000000..5f8b795 --- /dev/null +++ b/userland/ripgrep/.cargo-ok @@ -0,0 +1 @@ +{"v":1} \ No newline at end of file diff --git a/userland/ripgrep/.cargo/config.toml b/userland/ripgrep/.cargo/config.toml new file mode 100644 index 0000000..9e54301 --- /dev/null +++ b/userland/ripgrep/.cargo/config.toml @@ -0,0 +1,21 @@ +# On Windows MSVC, statically link the C runtime so that the resulting EXE does +# not depend on the vcruntime DLL. +# +# See: https://github.com/BurntSushi/ripgrep/pull/1613 +[target.x86_64-pc-windows-msvc] +rustflags = ["-C", "target-feature=+crt-static"] +[target.i686-pc-windows-msvc] +rustflags = ["-C", "target-feature=+crt-static"] + +# Do the same for MUSL targets. At the time of writing (2023-10-23), this is +# the default. But the plan is for the default to change to dynamic linking. +# The whole point of MUSL with respect to ripgrep is to create a fully +# statically linked executable. +# +# See: https://github.com/rust-lang/compiler-team/issues/422 +# See: https://github.com/rust-lang/compiler-team/issues/422#issuecomment-812135847 +[target.x86_64-unknown-linux-musl] +rustflags = [ + "-C", "target-feature=+crt-static", + "-C", "link-self-contained=yes", +] diff --git a/userland/ripgrep/.cargo_vcs_info.json b/userland/ripgrep/.cargo_vcs_info.json new file mode 100644 index 0000000..387c2f3 --- /dev/null +++ b/userland/ripgrep/.cargo_vcs_info.json @@ -0,0 +1,6 @@ +{ + "git": { + "sha1": "e89fff89ac9af12e8d4ce9d5fd07beb408ca730f" + }, + "path_in_vcs": "" +} \ No newline at end of file diff --git a/userland/ripgrep/.gitignore b/userland/ripgrep/.gitignore new file mode 100644 index 0000000..881633e --- /dev/null +++ b/userland/ripgrep/.gitignore @@ -0,0 +1,22 @@ +.*.swp +tags +target +/grep/Cargo.lock +/globset/Cargo.lock +/ignore/Cargo.lock +/termcolor/Cargo.lock +/wincolor/Cargo.lock +/deployment +/.idea + +# Snapcraft files +stage +prime +parts +*.snap +*.pyc +ripgrep*_source.tar.bz2 + +# Cargo timings +cargo-timing-*.html +cargo-timing.html diff --git a/userland/ripgrep/.ignore b/userland/ripgrep/.ignore new file mode 100644 index 0000000..1902214 --- /dev/null +++ b/userland/ripgrep/.ignore @@ -0,0 +1 @@ +!/.github/ diff --git a/userland/ripgrep/AI_POLICY.md b/userland/ripgrep/AI_POLICY.md new file mode 100644 index 0000000..6670269 --- /dev/null +++ b/userland/ripgrep/AI_POLICY.md @@ -0,0 +1,38 @@ +# AI Policy + +Using AI (i.e., LLMs) as tools for coding is welcome. A high bar is held for +all contributions to this project. Moreover, the project maintainers remain +responsible for any code that is published as part of a release. Contributors +are expected to be responsible for any code they publish. + +**AI should not be used to generate comments when communicating with +maintainers**. Comments are expected to be written by humans. Comments that are +believed to be written by AI may be hidden without notice. + +If you are opening an issue, you should be able to describe the problem in your +own words. + +If you are opening a pull request, you are expected to be able to explain the +proposed changes in your own words. This includes the pull request body and +responses to questions. **Do not copy responses from the AI when replying to +questions from maintainers.** + +This project requires a human in the loop who understands the work produced by +AI. **Autonomous agents are not allowed to be used for contributing to this +project**. Pull requests that appear in violation of this will be closed, +perhaps without notice. + +If you wish to include context from an interaction with AI in your comments, it +must be in a quote block (e.g., using `>`) and disclosed as such. It must be +accompanied by human commentary explaining the relevance and implications of +the context. Do not share long snippets. + +AI is useful when communicating as a non-native English speaker. If you are +using AI to edit your comments for this purpose, please take the time to ensure +it reflects your own voice and ideas. If using AI for translation, we recommend +writing in your native language and including the AI translation in a quote +block. + +This policy was adapted from [uv's AI policy]. + +[uv's AI policy]: https://github.com/astral-sh/.github/blob/c5187e200db51bfe11d56e13053d29bd3793fdd8/AI_POLICY.md diff --git a/userland/ripgrep/CHANGELOG.md b/userland/ripgrep/CHANGELOG.md new file mode 100644 index 0000000..8725cb8 --- /dev/null +++ b/userland/ripgrep/CHANGELOG.md @@ -0,0 +1,1864 @@ +15.2.0 (2026-07-15) +=================== +Unreleased changes. Release notes have not yet been written. + +Platform support: + +* `aarch64-unknown-linux-musl` is now included in the release binaries for + ripgrep. + +Performance improvements: + +* [PERF #3293](https://github.com/BurntSushi/ripgrep/issues/3293): + Improves directory traversal time on very large corpora. + +Feature enhancements: + +* [FEATURE #3275](https://github.com/BurntSushi/ripgrep/issues/3275): + ripgrep now respects `GIT_CONFIG_GLOBAL` and `GIT_CONFIG_SYSTEM`. + +Bug fixes: + +* [BUG #3212](https://github.com/BurntSushi/ripgrep/pull/3212): + Don't check for the existence of `.jj` when `--no-ignore` is used. +* [BUG #3320](https://github.com/BurntSushi/ripgrep/pull/3419), + [BUG #3376](https://github.com/BurntSushi/ripgrep/pull/3376), + [BUG #3419](https://github.com/BurntSushi/ripgrep/pull/3419): + Fix gitignore matching bug when searching across multiple directories. + + +15.1.0 (2025-10-22) +=================== +This is a small release that fixes a bug with how ripgrep handles line +buffering. This might manifest as ripgrep printing output later than you +expect or not working correctly with `tail -f` (even if you're using the +`--line-buffered` flag). + +Bug fixes: + +* [BUG #3194](https://github.com/BurntSushi/ripgrep/issues/3194): + Fix a regression with `--line-buffered` introduced in ripgrep 15.0.0. + +Feature enhancements: + +* [FEATURE #3192](https://github.com/BurntSushi/ripgrep/pull/3192): + Add hyperlink alias for Cursor. + + +15.0.0 (2025-10-15) +=================== +ripgrep 15 is a new major version release of ripgrep that mostly has bug fixes, +some minor performance improvements and minor new features. Here are some +highlights: + +* Several bugs around gitignore matching have been fixed. This includes + a commonly reported bug related to applying gitignore rules from parent + directories. +* A memory usage regression when handling very large gitignore files has been + fixed. +* `rg -vf file`, where `file` is empty, now matches everything. +* The `-r/--replace` flag now works with `--json`. +* A subset of Jujutsu (`jj`) repositories are now treated as if they were git + repositories. That is, ripgrep will respect `jj`'s gitignores. +* Globs can now use nested curly braces. + +Platform support: + +* `aarch64` for Windows now has release artifacts. +* `powerpc64` no longer has release artifacts generated for it. The CI + release workflow stopped working, and I didn't deem it worth my time to + debug it. If someone wants this and can test it, I'd be happy to add it + back. +* ripgrep binaries are now compiled with full LTO enabled. You may notice + small performance improvements from this and a modest decrease in binary + size. + +Performance improvements: + +* [PERF #2111](https://github.com/BurntSushi/ripgrep/issues/2111): + Don't resolve helper binaries on Windows when `-z/--search-zip` isn't used. +* [PERF #2865](https://github.com/BurntSushi/ripgrep/pull/2865): + Avoid using path canonicalization on Windows when emitting hyperlinks. + +Bug fixes: + +* [BUG #829](https://github.com/BurntSushi/ripgrep/issues/829), + [BUG #2731](https://github.com/BurntSushi/ripgrep/issues/2731), + [BUG #2747](https://github.com/BurntSushi/ripgrep/issues/2747), + [BUG #2770](https://github.com/BurntSushi/ripgrep/issues/2770), + [BUG #2778](https://github.com/BurntSushi/ripgrep/issues/2778), + [BUG #2836](https://github.com/BurntSushi/ripgrep/issues/2836), + [BUG #2933](https://github.com/BurntSushi/ripgrep/pull/2933), + [BUG #3067](https://github.com/BurntSushi/ripgrep/pull/3067): + Fix bug related to gitignores from parent directories. +* [BUG #1332](https://github.com/BurntSushi/ripgrep/issues/1332), + [BUG #3001](https://github.com/BurntSushi/ripgrep/issues/3001): + Make `rg -vf file` where `file` is empty match everything. +* [BUG #2177](https://github.com/BurntSushi/ripgrep/issues/2177): + Ignore a UTF-8 BOM marker at the start of `.gitignore` (and similar files). +* [BUG #2750](https://github.com/BurntSushi/ripgrep/issues/2750): + Fix memory usage regression for some truly large gitignore files. +* [BUG #2944](https://github.com/BurntSushi/ripgrep/pull/2944): + Fix a bug where the "bytes searched" in `--stats` output could be incorrect. +* [BUG #2990](https://github.com/BurntSushi/ripgrep/issues/2990): + Fix a bug where ripgrep would mishandle globs that ended with a `.`. +* [BUG #2094](https://github.com/BurntSushi/ripgrep/issues/2094), + [BUG #3076](https://github.com/BurntSushi/ripgrep/issues/3076): + Fix bug with `-m/--max-count` and `-U/--multiline` showing too many matches. +* [BUG #3100](https://github.com/BurntSushi/ripgrep/pull/3100): + Preserve line terminators when using `-r/--replace` flag. +* [BUG #3108](https://github.com/BurntSushi/ripgrep/issues/3108): + Fix a bug where `-q --files-without-match` inverted the exit code. +* [BUG #3131](https://github.com/BurntSushi/ripgrep/issues/3131): + Document inconsistency between `-c/--count` and `--files-with-matches`. +* [BUG #3135](https://github.com/BurntSushi/ripgrep/issues/3135): + Fix rare panic for some classes of large regexes on large haystacks. +* [BUG #3140](https://github.com/BurntSushi/ripgrep/issues/3140): + Ensure hyphens in flag names are escaped in the roff text for the man page. +* [BUG #3155](https://github.com/BurntSushi/ripgrep/issues/3155): + Statically compile PCRE2 into macOS release artifacts on `aarch64`. +* [BUG #3173](https://github.com/BurntSushi/ripgrep/issues/3173): + Fix ancestor ignore filter bug when searching whitelisted hidden files. +* [BUG #3178](https://github.com/BurntSushi/ripgrep/discussions/3178): + Fix bug causing incorrect summary statistics with `--json` flag. +* [BUG #3179](https://github.com/BurntSushi/ripgrep/issues/3179): + Fix gitignore bug when searching absolute paths with global gitignores. +* [BUG #3180](https://github.com/BurntSushi/ripgrep/issues/3180): + Fix a panicking bug when using `-U/--multiline` and `-r/--replace`. + +Feature enhancements: + +* Many enhancements to the default set of file types available for filtering. +* [FEATURE #1872](https://github.com/BurntSushi/ripgrep/issues/1872): + Make `-r/--replace` work with `--json`. +* [FEATURE #2708](https://github.com/BurntSushi/ripgrep/pull/2708): + Completions for the fish shell take ripgrep's config file into account. +* [FEATURE #2841](https://github.com/BurntSushi/ripgrep/pull/2841): + Add `italic` to the list of available style attributes in `--color`. +* [FEATURE #2842](https://github.com/BurntSushi/ripgrep/pull/2842): + Directories containing `.jj` are now treated as git repositories. +* [FEATURE #2849](https://github.com/BurntSushi/ripgrep/pull/2849): + When using multithreading, schedule files to search in order given on CLI. +* [FEATURE #2943](https://github.com/BurntSushi/ripgrep/issues/2943): + Add `aarch64` release artifacts for Windows. +* [FEATURE #3024](https://github.com/BurntSushi/ripgrep/issues/3024): + Add `highlight` color type, for styling non-matching text in a matching line. +* [FEATURE #3048](https://github.com/BurntSushi/ripgrep/pull/3048): + Globs in ripgrep (and the `globset` crate) now support nested alternates. +* [FEATURE #3096](https://github.com/BurntSushi/ripgrep/pull/3096): + Improve completions for `--hyperlink-format` in bash and fish. +* [FEATURE #3102](https://github.com/BurntSushi/ripgrep/pull/3102): + Improve completions for `--hyperlink-format` in zsh. + + +14.1.1 (2024-09-08) +=================== +This is a minor release with a bug fix for a matching bug. In particular, a bug +was found that could cause ripgrep to ignore lines that should match. That is, +false negatives. It is difficult to characterize the specific set of regexes +in which this occurs as it requires multiple different optimization strategies +to collide and produce an incorrect result. But as one reported example, in +ripgrep, the regex `(?i:e.x|ex)` does not match `e-x` when it should. (This +bug is a result of an inner literal optimization performed in the `grep-regex` +crate and not in the `regex` crate.) + +Bug fixes: + +* [BUG #2884](https://github.com/BurntSushi/ripgrep/issues/2884): + Fix bug where ripgrep could miss some matches that it should report. + +Miscellaneous: + +* [MISC #2748](https://github.com/BurntSushi/ripgrep/issues/2748): + Remove ripgrep's `simd-accel` feature because it was frequently broken. + + +14.1.0 (2024-01-06) +=================== +This is a minor release with a few small new features and bug fixes. This +release contains a bug fix for unbounded memory growth while walking a +directory tree. This release also includes improvements to the completions for +the `fish` shell, and release binaries for several additional ARM targets. + +Bug fixes: + +* [BUG #2664](https://github.com/BurntSushi/ripgrep/issues/2690): + Fix unbounded memory growth in the `ignore` crate. + +Feature enhancements: + +* Added or improved file type filtering for Lean and Meson. +* [FEATURE #2684](https://github.com/BurntSushi/ripgrep/issues/2684): + Improve completions for the `fish` shell. +* [FEATURE #2702](https://github.com/BurntSushi/ripgrep/pull/2702): + Add release binaries for `armv7-unknown-linux-gnueabihf`, + `armv7-unknown-linux-musleabihf` and `armv7-unknown-linux-musleabi`. + + +14.0.3 (2023-11-28) +=================== +This is a patch release with a bug fix for the `--sortr` flag. + +Bug fixes: + +* [BUG #2664](https://github.com/BurntSushi/ripgrep/issues/2664): + Fix `--sortr=path`. I left a `todo!()` in the source. Oof. + + +14.0.2 (2023-11-27) +=================== +This is a patch release with a few small bug fixes. + +Bug fixes: + +* [BUG #2654](https://github.com/BurntSushi/ripgrep/issues/2654): + Fix `deb` release sha256 sum file. +* [BUG #2658](https://github.com/BurntSushi/ripgrep/issues/2658): + Fix partial regression in the behavior of `--null-data --line-regexp`. +* [BUG #2659](https://github.com/BurntSushi/ripgrep/issues/2659): + Fix Fish shell completions. +* [BUG #2662](https://github.com/BurntSushi/ripgrep/issues/2662): + Fix typo in documentation for `-i/--ignore-case`. + + +14.0.1 (2023-11-26) +=================== +This a patch release meant to fix `cargo install ripgrep` on Windows. + +Bug fixes: + +* [BUG #2653](https://github.com/BurntSushi/ripgrep/issues/2653): + Include `pkg/windows/Manifest.xml` in crate package. + + +14.0.0 (2023-11-26) +=================== +ripgrep 14 is a new major version release of ripgrep that has some new +features, performance improvements and a lot of bug fixes. + +The headlining feature in this release is hyperlink support. In this release, +they are an opt-in feature but may change to an opt-out feature in the future. +To enable them, try passing `--hyperlink-format default`. If you use [VS Code], +then try passing `--hyperlink-format vscode`. Please [report your experience +with hyperlinks][report-hyperlinks], positive or negative. + +[VS Code]: https://code.visualstudio.com/ +[report-hyperlinks]: https://github.com/BurntSushi/ripgrep/discussions/2611 + +Another headlining development in this release is that it contains a rewrite +of its regex engine. You generally shouldn't notice any changes, except for +some searches may get faster. You can read more about the [regex engine rewrite +on my blog][regex-internals]. Please [report your performance improvements or +regressions that you notice][report-perf]. + +[report-perf]: https://github.com/BurntSushi/ripgrep/discussions/2652 + +Finally, ripgrep switched the library it uses for argument parsing. Users +should not notice a difference in most cases (error messages have changed +somewhat), but flag overrides should generally be more consistent. For example, +things like `--no-ignore --ignore-vcs` work as one would expect (disables all +filtering related to ignore rules except for rules found in version control +systems such as `git`). + +[regex-internals]: https://blog.burntsushi.net/regex-internals/ + +**BREAKING CHANGES**: + +* `rg -C1 -A2` used to be equivalent to `rg -A2`, but now it is equivalent to + `rg -B1 -A2`. That is, `-A` and `-B` no longer completely override `-C`. + Instead, they only partially override `-C`. + +Build process changes: + +* ripgrep's shell completions and man page are now created by running ripgrep +with a new `--generate` flag. For example, `rg --generate man` will write a +man page in `roff` format on stdout. The release archives have not changed. +* The optional build dependency on `asciidoc` or `asciidoctor` has been +dropped. Previously, it was used to produce ripgrep's man page. ripgrep now +owns this process itself by writing `roff` directly. + +Performance improvements: + +* [PERF #1746](https://github.com/BurntSushi/ripgrep/issues/1746): + Make some cases with inner literals faster. +* [PERF #1760](https://github.com/BurntSushi/ripgrep/issues/1760): + Make most searches with `\b` look-arounds (among others) much faster. +* [PERF #2591](https://github.com/BurntSushi/ripgrep/pull/2591): + Parallel directory traversal now uses work stealing for faster searches. +* [PERF #2642](https://github.com/BurntSushi/ripgrep/pull/2642): + Parallel directory traversal has some contention reduced. + +Feature enhancements: + +* Added or improved file type filtering for Ada, DITA, Elixir, Fuchsia, Gentoo, + Gradle, GraphQL, Markdown, Prolog, Raku, TypeScript, USD, V +* [FEATURE #665](https://github.com/BurntSushi/ripgrep/issues/665): + Add a new `--hyperlink-format` flag that turns file paths into hyperlinks. +* [FEATURE #1709](https://github.com/BurntSushi/ripgrep/issues/1709): + Improve documentation of ripgrep's behavior when stdout is a tty. +* [FEATURE #1737](https://github.com/BurntSushi/ripgrep/issues/1737): + Provide binaries for Apple silicon. +* [FEATURE #1790](https://github.com/BurntSushi/ripgrep/issues/1790): + Add new `--stop-on-nonmatch` flag. +* [FEATURE #1814](https://github.com/BurntSushi/ripgrep/issues/1814): + Flags are now categorized in `-h/--help` output and ripgrep's man page. +* [FEATURE #1838](https://github.com/BurntSushi/ripgrep/issues/1838): + An error is shown when searching for NUL bytes with binary detection enabled. +* [FEATURE #2195](https://github.com/BurntSushi/ripgrep/issues/2195): + When `extra-verbose` mode is enabled in zsh, show extra file type info. +* [FEATURE #2298](https://github.com/BurntSushi/ripgrep/issues/2298): + Add instructions for installing ripgrep using `cargo binstall`. +* [FEATURE #2409](https://github.com/BurntSushi/ripgrep/pull/2409): + Added installation instructions for `winget`. +* [FEATURE #2425](https://github.com/BurntSushi/ripgrep/pull/2425): + Shell completions (and man page) can be created via `rg --generate`. +* [FEATURE #2524](https://github.com/BurntSushi/ripgrep/issues/2524): + The `--debug` flag now indicates whether stdin or `./` is being searched. +* [FEATURE #2643](https://github.com/BurntSushi/ripgrep/issues/2643): + Make `-d` a short flag for `--max-depth`. +* [FEATURE #2645](https://github.com/BurntSushi/ripgrep/issues/2645): + The `--version` output will now also contain PCRE2 availability information. + +Bug fixes: + +* [BUG #884](https://github.com/BurntSushi/ripgrep/issues/884): + Don't error when `-v/--invert-match` is used multiple times. +* [BUG #1275](https://github.com/BurntSushi/ripgrep/issues/1275): + Fix bug with `\b` assertion in the regex engine. +* [BUG #1376](https://github.com/BurntSushi/ripgrep/issues/1376): + Using `--no-ignore --ignore-vcs` now works as one would expect. +* [BUG #1622](https://github.com/BurntSushi/ripgrep/issues/1622): + Add note about error messages to `-z/--search-zip` documentation. +* [BUG #1648](https://github.com/BurntSushi/ripgrep/issues/1648): + Fix bug where sometimes short flags with values, e.g., `-M 900`, would fail. +* [BUG #1701](https://github.com/BurntSushi/ripgrep/issues/1701): + Fix bug where some flags could not be repeated. +* [BUG #1757](https://github.com/BurntSushi/ripgrep/issues/1757): + Fix bug when searching a sub-directory didn't have ignores applied correctly. +* [BUG #1891](https://github.com/BurntSushi/ripgrep/issues/1891): + Fix bug when using `-w` with a regex that can match the empty string. +* [BUG #1911](https://github.com/BurntSushi/ripgrep/issues/1911): + Disable mmap searching in all non-64-bit environments. +* [BUG #1966](https://github.com/BurntSushi/ripgrep/issues/1966): + Fix bug where ripgrep can panic when printing to stderr. +* [BUG #2046](https://github.com/BurntSushi/ripgrep/issues/2046): + Clarify that `--pre` can accept any kind of path in the documentation. +* [BUG #2108](https://github.com/BurntSushi/ripgrep/issues/2108): + Improve docs for `-r/--replace` syntax. +* [BUG #2198](https://github.com/BurntSushi/ripgrep/issues/2198): + Fix bug where `--no-ignore-dot` would not ignore `.rgignore`. +* [BUG #2201](https://github.com/BurntSushi/ripgrep/issues/2201): + Improve docs for `-r/--replace` flag. +* [BUG #2288](https://github.com/BurntSushi/ripgrep/issues/2288): + `-A` and `-B` now only each partially override `-C`. +* [BUG #2236](https://github.com/BurntSushi/ripgrep/issues/2236): + Fix gitignore parsing bug where a trailing `\/` resulted in an error. +* [BUG #2243](https://github.com/BurntSushi/ripgrep/issues/2243): + Fix `--sort` flag for values other than `path`. +* [BUG #2246](https://github.com/BurntSushi/ripgrep/issues/2246): + Add note in `--debug` logs when binary files are ignored. +* [BUG #2337](https://github.com/BurntSushi/ripgrep/issues/2337): + Improve docs to mention that `--stats` is always implied by `--json`. +* [BUG #2381](https://github.com/BurntSushi/ripgrep/issues/2381): + Make `-p/--pretty` override flags like `--no-line-number`. +* [BUG #2392](https://github.com/BurntSushi/ripgrep/issues/2392): + Improve global git config parsing of the `excludesFile` field. +* [BUG #2418](https://github.com/BurntSushi/ripgrep/pull/2418): + Clarify sorting semantics of `--sort=path`. +* [BUG #2458](https://github.com/BurntSushi/ripgrep/pull/2458): + Make `--trim` run before `-M/--max-columns` takes effect. +* [BUG #2479](https://github.com/BurntSushi/ripgrep/issues/2479): + Add documentation about `.ignore`/`.rgignore` files in parent directories. +* [BUG #2480](https://github.com/BurntSushi/ripgrep/issues/2480): + Fix bug when using inline regex flags with `-e/--regexp`. +* [BUG #2505](https://github.com/BurntSushi/ripgrep/issues/2505): + Improve docs for `--vimgrep` by mentioning footguns and some work-arounds. +* [BUG #2519](https://github.com/BurntSushi/ripgrep/issues/2519): + Fix incorrect default value in documentation for `--field-match-separator`. +* [BUG #2523](https://github.com/BurntSushi/ripgrep/issues/2523): + Make executable searching take `.com` into account on Windows. +* [BUG #2574](https://github.com/BurntSushi/ripgrep/issues/2574): + Fix bug in `-w/--word-regexp` that would result in incorrect match offsets. +* [BUG #2623](https://github.com/BurntSushi/ripgrep/issues/2623): + Fix a number of bugs with the `-w/--word-regexp` flag. +* [BUG #2636](https://github.com/BurntSushi/ripgrep/pull/2636): + Strip release binaries for macOS. + + +13.0.0 (2021-06-12) +=================== +ripgrep 13 is a new major version release of ripgrep that primarily contains +bug fixes, some performance improvements and a few minor breaking changes. +There is also a fix for a security vulnerability on Windows +([CVE-2021-3013](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-3013)). + +Some highlights: + +A new short flag, `-.`, has been added. It is an alias for the `--hidden` flag, +which instructs ripgrep to search hidden files and directories. + +ripgrep is now using a new +[vectorized implementation of `memmem`](https://github.com/BurntSushi/memchr/pull/82), +which accelerates many common searches. If you notice any performance +regressions (or major improvements), I'd love to hear about them through an +issue report! + +Also, for Windows users targeting MSVC, Cargo will now build fully static +executables of ripgrep. The release binaries for ripgrep 13 have been compiled +using this configuration. + +**BREAKING CHANGES**: + +**Binary detection output has changed slightly.** + +In this release, a small tweak has been made to the output format when a binary +file is detected. Previously, it looked like this: + +``` +Binary file FOO matches (found "\0" byte around offset XXX) +``` + +Now it looks like this: + +``` +FOO: binary file matches (found "\0" byte around offset XXX) +``` + +**vimgrep output in multi-line now only prints the first line for each match.** + +See [issue 1866](https://github.com/BurntSushi/ripgrep/issues/1866) for more +discussion on this. Previously, every line in a match was duplicated, even +when it spanned multiple lines. There are no changes to vimgrep output when +multi-line mode is disabled. + +**In multi-line mode, --count is now equivalent to --count-matches.** + +This appears to match how `pcre2grep` implements `--count`. Previously, ripgrep +would produce outright incorrect counts. Another alternative would be to simply +count the number of lines---even if it's more than the number of matches---but +that seems highly unintuitive. + +**FULL LIST OF FIXES AND IMPROVEMENTS:** + +Security fixes: + +* [CVE-2021-3013](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2021-3013): + Fixes a security hole on Windows where running ripgrep with either the + `-z/--search-zip` or `--pre` flags can result in running arbitrary + executables from the current directory. +* [VULN #1773](https://github.com/BurntSushi/ripgrep/issues/1773): + This is the public facing issue tracking CVE-2021-3013. ripgrep's README + now contains a section describing how to report a vulnerability. + +Performance improvements: + +* [PERF #1657](https://github.com/BurntSushi/ripgrep/discussions/1657): + Check if a file should be ignored first before issuing stat calls. +* [PERF memchr#82](https://github.com/BurntSushi/memchr/pull/82): + ripgrep now uses a new vectorized implementation of `memmem`. + +Feature enhancements: + +* Added or improved file type filtering for ASP, Bazel, dvc, FlatBuffers, + Futhark, minified files, Mint, pofiles (from GNU gettext) Racket, Red, Ruby, + VCL, Yang. +* [FEATURE #1404](https://github.com/BurntSushi/ripgrep/pull/1404): + ripgrep now prints a warning if nothing is searched. +* [FEATURE #1613](https://github.com/BurntSushi/ripgrep/pull/1613): + Cargo will now produce static executables on Windows when using MSVC. +* [FEATURE #1680](https://github.com/BurntSushi/ripgrep/pull/1680): + Add `-.` as a short flag alias for `--hidden`. +* [FEATURE #1842](https://github.com/BurntSushi/ripgrep/issues/1842): + Add `--field-{context,match}-separator` for customizing field delimiters. +* [FEATURE #1856](https://github.com/BurntSushi/ripgrep/pull/1856): + The README now links to a + [Spanish translation](https://github.com/UltiRequiem/traducciones/tree/master/ripgrep). + +Bug fixes: + +* [BUG #1277](https://github.com/BurntSushi/ripgrep/issues/1277): + Document cygwin path translation behavior in the FAQ. +* [BUG #1739](https://github.com/BurntSushi/ripgrep/issues/1739): + Fix bug where replacements were buggy if the regex matched a line terminator. +* [BUG #1311](https://github.com/BurntSushi/ripgrep/issues/1311): + Fix multi-line bug where a search & replace for `\n` didn't work as expected. +* [BUG #1401](https://github.com/BurntSushi/ripgrep/issues/1401): + Fix buggy interaction between PCRE2 look-around and `-o/--only-matching`. +* [BUG #1412](https://github.com/BurntSushi/ripgrep/issues/1412): + Fix multi-line bug with searches using look-around past matching lines. +* [BUG #1577](https://github.com/BurntSushi/ripgrep/issues/1577): + Fish shell completions will continue to be auto-generated. +* [BUG #1642](https://github.com/BurntSushi/ripgrep/issues/1642): + Fixes a bug where using `-m` and `-A` printed more matches than the limit. +* [BUG #1703](https://github.com/BurntSushi/ripgrep/issues/1703): + Clarify the function of `-u/--unrestricted`. +* [BUG #1708](https://github.com/BurntSushi/ripgrep/issues/1708): + Clarify how `-S/--smart-case` works. +* [BUG #1730](https://github.com/BurntSushi/ripgrep/issues/1730): + Clarify that CLI invocation must always be valid, regardless of config file. +* [BUG #1741](https://github.com/BurntSushi/ripgrep/issues/1741): + Fix stdin detection when using PowerShell in UNIX environments. +* [BUG #1756](https://github.com/BurntSushi/ripgrep/pull/1756): + Fix bug where `foo/**` would match `foo`, but it shouldn't. +* [BUG #1765](https://github.com/BurntSushi/ripgrep/issues/1765): + Fix panic when `--crlf` is used in some cases. +* [BUG #1638](https://github.com/BurntSushi/ripgrep/issues/1638): + Correctly sniff UTF-8 and do transcoding, like we do for UTF-16. +* [BUG #1816](https://github.com/BurntSushi/ripgrep/issues/1816): + Add documentation for glob alternate syntax, e.g., `{a,b,..}`. +* [BUG #1847](https://github.com/BurntSushi/ripgrep/issues/1847): + Clarify how the `--hidden` flag works. +* [BUG #1866](https://github.com/BurntSushi/ripgrep/issues/1866#issuecomment-841635553): + Fix bug when computing column numbers in `--vimgrep` mode. +* [BUG #1868](https://github.com/BurntSushi/ripgrep/issues/1868): + Fix bug where `--passthru` and `-A/-B/-C` did not override each other. +* [BUG #1869](https://github.com/BurntSushi/ripgrep/pull/1869): + Clarify docs for `--files-with-matches` and `--files-without-match`. +* [BUG #1878](https://github.com/BurntSushi/ripgrep/issues/1878): + Fix bug where `\A` could produce unanchored matches in multiline search. +* [BUG 94e4b8e3](https://github.com/BurntSushi/ripgrep/commit/94e4b8e3): + Fix column numbers with `--vimgrep` is used with `-U/--multiline`. + + +12.1.1 (2020-05-29) +=================== +ripgrep 12.1.1 is a patch release that fixes a couple small bugs. In +particular, the ripgrep 12.1.0 release did not tag new releases for all of its +in-tree dependencies. As a result, ripgrep built dependencies from crates.io +would produce a different build than compiling ripgrep from source on the +`12.1.0` tag. Namely, some crates like `grep-cli` had unreleased changes. + +Bug fixes: + +* [BUG #1581](https://github.com/BurntSushi/ripgrep/issues/1581): + Corrects some egregious markup output in `--help`. +* [BUG #1591](https://github.com/BurntSushi/ripgrep/issues/1591): + Mention the special `$0` capture group in docs for the `-r/--replace` flag. +* [BUG #1602](https://github.com/BurntSushi/ripgrep/issues/1602): + Fix failing test resulting from out-of-sync dependencies. + + +12.1.0 (2020-05-09) +=================== +ripgrep 12.1.0 is a small minor version release that mostly includes bug fixes +and documentation improvements. This release also contains some important +notices for downstream packagers. + +**Notices for downstream ripgrep package maintainers:** + +* Fish shell completions will be removed in the ripgrep 13 release. + See [#1577](https://github.com/BurntSushi/ripgrep/issues/1577) + for more details. +* ripgrep has switched from `a2x` to `asciidoctor` to generate the man page. + If `asciidoctor` is not present, then ripgrep will currently fall back to + `a2x`. Support for `a2x` will be dropped in the ripgrep 13 release. + See [#1544](https://github.com/BurntSushi/ripgrep/issues/1544) + for more details. + +Feature enhancements: + +* [FEATURE #1547](https://github.com/BurntSushi/ripgrep/pull/1547): + Support decompressing `.Z` files via `uncompress`. + +Bug fixes: + +* [BUG #1252](https://github.com/BurntSushi/ripgrep/issues/1252): + Add a section on the `--pre` flag to the GUIDE. +* [BUG #1339](https://github.com/BurntSushi/ripgrep/issues/1339): + Improve error message when a pattern with invalid UTF-8 is provided. +* [BUG #1524](https://github.com/BurntSushi/ripgrep/issues/1524): + Note how to escape a `$` when using `--replace`. +* [BUG #1537](https://github.com/BurntSushi/ripgrep/issues/1537): + Fix match bug caused by inner literal optimization. +* [BUG #1544](https://github.com/BurntSushi/ripgrep/issues/1544): + ripgrep now uses `asciidoctor` instead of `a2x` to generate its man page. +* [BUG #1550](https://github.com/BurntSushi/ripgrep/issues/1550): + Substantially reduce peak memory usage when searching wide directories. +* [BUG #1571](https://github.com/BurntSushi/ripgrep/issues/1571): + Add note about configuration files in `--type-{add,clear}` docs. +* [BUG #1573](https://github.com/BurntSushi/ripgrep/issues/1573): + Fix incorrect `--count-matches` output when using look-around. + + +12.0.1 (2020-03-29) +=================== +ripgrep 12.0.1 is a small patch release that includes a minor bug fix relating +to superfluous error messages when searching git repositories with sub-modules. +This was a regression introduced in the 12.0.0 release. + +Bug fixes: + +* [BUG #1520](https://github.com/BurntSushi/ripgrep/issues/1520): + Don't emit spurious error messages in git repositories with submodules. + + +12.0.0 (2020-03-15) +=================== +ripgrep 12 is a new major version release of ripgrep that contains many bug +fixes, several important performance improvements and a few minor new features. + +In a near future release, I am hoping to add an +[indexing feature](https://github.com/BurntSushi/ripgrep/issues/1497) +to ripgrep, which will dramatically speed up searching by building an index. +Feedback would very much be appreciated, especially on the user experience +which will be difficult to get right. + +This release has no known breaking changes. + +Deprecations: + +* The `--no-pcre2-unicode` flag is deprecated. Instead, use the `--no-unicode` + flag, which applies to both the default regex engine and PCRE2. For now, + `--no-pcre2-unicode` and `--pcre2-unicode` are aliases to `--no-unicode` + and `--unicode`, respectively. The `--[no-]pcre2-unicode` flags may be + removed in a future release. +* The `--auto-hybrid-regex` flag is deprecated. Instead, use the new `--engine` + flag with the `auto` value. + +Performance improvements: + +* [PERF #1087](https://github.com/BurntSushi/ripgrep/pull/1087): + ripgrep is smarter when detected literals are whitespace. +* [PERF #1381](https://github.com/BurntSushi/ripgrep/pull/1381): + Directory traversal is sped up with speculative ignore-file existence checks. +* [PERF cd8ec38a](https://github.com/BurntSushi/ripgrep/commit/cd8ec38a): + Improve inner literal detection to cover more cases more effectively. + e.g., ` +Sherlock Holmes +` now has ` Sherlock Holmes ` extracted instead + of ` `. +* [PERF 6a0e0147](https://github.com/BurntSushi/ripgrep/commit/6a0e0147): + Improve literal detection when the `-w/--word-regexp` flag is used. +* [PERF ad97e9c9](https://github.com/BurntSushi/ripgrep/commit/ad97e9c9): + Improve overall performance of the `-w/--word-regexp` flag. + +Feature enhancements: + +* Added or improved file type filtering for erb, diff, Gradle, HAML, Org, + Postscript, Skim, Slim, Slime, RPM Spec files, Typoscript, xml. +* [FEATURE #1370](https://github.com/BurntSushi/ripgrep/pull/1370): + Add `--include-zero` flag that shows files searched without matches. +* [FEATURE #1390](https://github.com/BurntSushi/ripgrep/pull/1390): + Add `--no-context-separator` flag that always hides context separators. +* [FEATURE #1414](https://github.com/BurntSushi/ripgrep/pull/1414): + Add `--no-require-git` flag to allow ripgrep to respect gitignores anywhere. +* [FEATURE #1420](https://github.com/BurntSushi/ripgrep/pull/1420): + Add `--no-ignore-exclude` to disregard rules in `.git/info/exclude` files. +* [FEATURE #1466](https://github.com/BurntSushi/ripgrep/pull/1466): + Add `--no-ignore-files` flag to disable all `--ignore-file` flags. +* [FEATURE #1488](https://github.com/BurntSushi/ripgrep/pull/1488): + Add '--engine' flag for easier switching between regex engines. +* [FEATURE 75cbe88f](https://github.com/BurntSushi/ripgrep/commit/75cbe88f): + Add `--no-unicode` flag. This works on all supported regex engines. + +Bug fixes: + +* [BUG #1291](https://github.com/BurntSushi/ripgrep/issues/1291): + ripgrep now works in non-existent directories. +* [BUG #1319](https://github.com/BurntSushi/ripgrep/issues/1319): + Fix match bug due to errant literal detection. +* [**BUG #1335**](https://github.com/BurntSushi/ripgrep/issues/1335): + Fixes a performance bug when searching plain text files with very long lines. + This was a serious performance regression in some cases. +* [BUG #1344](https://github.com/BurntSushi/ripgrep/issues/1344): + Document usage of `--type all`. +* [BUG #1389](https://github.com/BurntSushi/ripgrep/issues/1389): + Fixes a bug where ripgrep would panic when searching a symlinked directory. +* [BUG #1439](https://github.com/BurntSushi/ripgrep/issues/1439): + Improve documentation for ripgrep's automatic stdin detection. +* [BUG #1441](https://github.com/BurntSushi/ripgrep/issues/1441): + Remove CPU features from man page. +* [BUG #1442](https://github.com/BurntSushi/ripgrep/issues/1442), + [BUG #1478](https://github.com/BurntSushi/ripgrep/issues/1478): + Improve documentation of the `-g/--glob` flag. +* [BUG #1445](https://github.com/BurntSushi/ripgrep/issues/1445): + ripgrep now respects ignore rules from .git/info/exclude in worktrees. +* [BUG #1485](https://github.com/BurntSushi/ripgrep/issues/1485): + Fish shell completions from the release Debian package are now installed to + `/usr/share/fish/vendor_completions.d/rg.fish`. + + +11.0.2 (2019-08-01) +=================== +ripgrep 11.0.2 is a new patch release that fixes a few bugs, including a +performance regression and a matching bug when using the `-F/--fixed-strings` +flag. + +Feature enhancements: + +* [FEATURE #1293](https://github.com/BurntSushi/ripgrep/issues/1293): + Added `--glob-case-insensitive` flag that makes `--glob` behave as `--iglob`. + +Bug fixes: + +* [BUG #1246](https://github.com/BurntSushi/ripgrep/issues/1246): + Add translations to README, starting with an unofficial Chinese translation. +* [BUG #1259](https://github.com/BurntSushi/ripgrep/issues/1259): + Fix bug where the last byte of a `-f file` was stripped if it wasn't a `\n`. +* [BUG #1261](https://github.com/BurntSushi/ripgrep/issues/1261): + Document that no error is reported when searching for `\n` with `-P/--pcre2`. +* [BUG #1284](https://github.com/BurntSushi/ripgrep/issues/1284): + Mention `.ignore` and `.rgignore` more prominently in the README. +* [BUG #1292](https://github.com/BurntSushi/ripgrep/issues/1292): + Fix bug where `--with-filename` was sometimes enabled incorrectly. +* [BUG #1268](https://github.com/BurntSushi/ripgrep/issues/1268): + Fix major performance regression in GitHub `x86_64-linux` binary release. +* [BUG #1302](https://github.com/BurntSushi/ripgrep/issues/1302): + Show better error messages when a non-existent preprocessor command is given. +* [BUG #1334](https://github.com/BurntSushi/ripgrep/issues/1334): + Fix match regression with `-F` flag when patterns contain meta characters. + + +11.0.1 (2019-04-16) +=================== +ripgrep 11.0.1 is a new patch release that fixes a search regression introduced +in the previous 11.0.0 release. In particular, ripgrep can enter an infinite +loop for some search patterns when searching invalid UTF-8. + +Bug fixes: + +* [BUG #1247](https://github.com/BurntSushi/ripgrep/issues/1247): + Fix search bug that can cause ripgrep to enter an infinite loop. + + +11.0.0 (2019-04-15) +=================== +ripgrep 11 is a new major version release of ripgrep that contains many bug +fixes, some performance improvements and a few feature enhancements. Notably, +ripgrep's user experience for binary file filtering has been improved. See the +[guide's new section on binary data](GUIDE.md#binary-data) for more details. + +This release also marks a change in ripgrep's versioning. Where as the previous +version was `0.10.0`, this version is `11.0.0`. Moving forward, ripgrep's +major version will be increased a few times per year. ripgrep will continue to +be conservative with respect to backwards compatibility, but may occasionally +introduce breaking changes, which will always be documented in this CHANGELOG. +See [issue 1172](https://github.com/BurntSushi/ripgrep/issues/1172) for a bit +more detail on why this versioning change was made. + +This release increases the **minimum supported Rust version** from 1.28.0 to +1.34.0. + +**BREAKING CHANGES**: + +* ripgrep has tweaked its exit status codes to be more like GNU grep's. Namely, + if a non-fatal error occurs during a search, then ripgrep will now always + emit a `2` exit status code, regardless of whether a match is found or not. + Previously, ripgrep would only emit a `2` exit status code for a catastrophic + error (e.g., regex syntax error). One exception to this is if ripgrep is run + with `-q/--quiet`. In that case, if an error occurs and a match is found, + then ripgrep will exit with a `0` exit status code. +* Supplying the `-u/--unrestricted` flag three times is now equivalent to + supplying `--no-ignore --hidden --binary`. Previously, `-uuu` was equivalent + to `--no-ignore --hidden --text`. The difference is that `--binary` disables + binary file filtering without potentially dumping binary data into your + terminal. That is, `rg -uuu foo` should now be equivalent to `grep -r foo`. +* The `avx-accel` feature of ripgrep has been removed since it is no longer + necessary. All uses of AVX in ripgrep are now enabled automatically via + runtime CPU feature detection. The `simd-accel` feature does remain available + (only for enabling SIMD for transcoding), however, it does increase + compilation times substantially at the moment. + +Performance improvements: + +* [PERF #497](https://github.com/BurntSushi/ripgrep/issues/497), + [PERF #838](https://github.com/BurntSushi/ripgrep/issues/838): + Make `rg -F -f dictionary-of-literals` much faster. + +Feature enhancements: + +* Added or improved file type filtering for Apache Thrift, ASP, Bazel, Brotli, + BuildStream, bzip2, C, C++, Cython, gzip, Java, Make, Postscript, QML, Tex, + XML, xz, zig and zstd. +* [FEATURE #855](https://github.com/BurntSushi/ripgrep/issues/855): + Add `--binary` flag for disabling binary file filtering. +* [FEATURE #1078](https://github.com/BurntSushi/ripgrep/pull/1078): + Add `--max-columns-preview` flag for showing a preview of long lines. +* [FEATURE #1099](https://github.com/BurntSushi/ripgrep/pull/1099): + Add support for Brotli and Zstd to the `-z/--search-zip` flag. +* [FEATURE #1138](https://github.com/BurntSushi/ripgrep/pull/1138): + Add `--no-ignore-dot` flag for ignoring `.ignore` files. +* [FEATURE #1155](https://github.com/BurntSushi/ripgrep/pull/1155): + Add `--auto-hybrid-regex` flag for automatically falling back to PCRE2. +* [FEATURE #1159](https://github.com/BurntSushi/ripgrep/pull/1159): + ripgrep's exit status logic should now match GNU grep. See updated man page. +* [FEATURE #1164](https://github.com/BurntSushi/ripgrep/pull/1164): + Add `--ignore-file-case-insensitive` for case insensitive ignore globs. +* [FEATURE #1185](https://github.com/BurntSushi/ripgrep/pull/1185): + Add `-I` flag as a short option for the `--no-filename` flag. +* [FEATURE #1207](https://github.com/BurntSushi/ripgrep/pull/1207): + Add `none` value to `-E/--encoding` to forcefully disable all transcoding. +* [FEATURE da9d7204](https://github.com/BurntSushi/ripgrep/commit/da9d7204): + Add `--pcre2-version` for querying showing PCRE2 version information. + +Bug fixes: + +* [BUG #306](https://github.com/BurntSushi/ripgrep/issues/306), + [BUG #855](https://github.com/BurntSushi/ripgrep/issues/855): + Improve the user experience for ripgrep's binary file filtering. +* [BUG #373](https://github.com/BurntSushi/ripgrep/issues/373), + [BUG #1098](https://github.com/BurntSushi/ripgrep/issues/1098): + `**` is now accepted as valid syntax anywhere in a glob. +* [BUG #916](https://github.com/BurntSushi/ripgrep/issues/916): + ripgrep no longer hangs when searching `/proc` with a zombie process present. +* [BUG #1052](https://github.com/BurntSushi/ripgrep/issues/1052): + Fix bug where ripgrep could panic when transcoding UTF-16 files. +* [BUG #1055](https://github.com/BurntSushi/ripgrep/issues/1055): + Suggest `-U/--multiline` when a pattern contains a `\n`. +* [BUG #1063](https://github.com/BurntSushi/ripgrep/issues/1063): + Always strip a BOM if it's present, even for UTF-8. +* [BUG #1064](https://github.com/BurntSushi/ripgrep/issues/1064): + Fix inner literal detection that could lead to incorrect matches. +* [BUG #1079](https://github.com/BurntSushi/ripgrep/issues/1079): + Fixes a bug where the order of globs could result in missing a match. +* [BUG #1089](https://github.com/BurntSushi/ripgrep/issues/1089): + Fix another bug where ripgrep could panic when transcoding UTF-16 files. +* [BUG #1091](https://github.com/BurntSushi/ripgrep/issues/1091): + Add note about inverted flags to the man page. +* [BUG #1093](https://github.com/BurntSushi/ripgrep/pull/1093): + Fix handling of literal slashes in gitignore patterns. +* [BUG #1095](https://github.com/BurntSushi/ripgrep/issues/1095): + Fix corner cases involving the `--crlf` flag. +* [BUG #1101](https://github.com/BurntSushi/ripgrep/issues/1101): + Fix AsciiDoc escaping for man page output. +* [BUG #1103](https://github.com/BurntSushi/ripgrep/issues/1103): + Clarify what `--encoding auto` does. +* [BUG #1106](https://github.com/BurntSushi/ripgrep/issues/1106): + `--files-with-matches` and `--files-without-match` work with one file. +* [BUG #1121](https://github.com/BurntSushi/ripgrep/issues/1121): + Fix bug that was triggering Windows antimalware when using the `--files` + flag. +* [BUG #1125](https://github.com/BurntSushi/ripgrep/issues/1125), + [BUG #1159](https://github.com/BurntSushi/ripgrep/issues/1159): + ripgrep shouldn't panic for `rg -h | rg` and should emit correct exit status. +* [BUG #1144](https://github.com/BurntSushi/ripgrep/issues/1144): + Fixes a bug where line numbers could be wrong on big-endian machines. +* [BUG #1154](https://github.com/BurntSushi/ripgrep/issues/1154): + Windows files with "hidden" attribute are now treated as hidden. +* [BUG #1173](https://github.com/BurntSushi/ripgrep/issues/1173): + Fix handling of `**` patterns in gitignore files. +* [BUG #1174](https://github.com/BurntSushi/ripgrep/issues/1174): + Fix handling of repeated `**` patterns in gitignore files. +* [BUG #1176](https://github.com/BurntSushi/ripgrep/issues/1176): + Fix bug where `-F`/`-x` weren't applied to patterns given via `-f`. +* [BUG #1189](https://github.com/BurntSushi/ripgrep/issues/1189): + Document cases where ripgrep may use a lot of memory. +* [BUG #1203](https://github.com/BurntSushi/ripgrep/issues/1203): + Fix a matching bug related to the suffix literal optimization. +* [BUG 8f14cb18](https://github.com/BurntSushi/ripgrep/commit/8f14cb18): + Increase the default stack size for PCRE2's JIT. + + +0.10.0 (2018-09-07) +=================== +This is a new minor version release of ripgrep that contains some major new +features, a huge number of bug fixes, and is the first release based on +libripgrep. The entirety of ripgrep's core search and printing code has been +rewritten and generalized so that anyone can make use of it. + +Major new features include PCRE2 support, multi-line search and a JSON output +format. + +**BREAKING CHANGES**: + +* The minimum version required to compile Rust has now changed to track the + latest stable version of Rust. Patch releases will continue to compile with + the same version of Rust as the previous patch release, but new minor + versions will use the current stable version of the Rust compile as its + minimum supported version. +* The match semantics of `-w/--word-regexp` have changed slightly. They used + to be `\b(?:)\b`, but now it's + `(?:^|\W)(?:)(?:$|\W)`. This matches the behavior of GNU grep + and is believed to be closer to the intended semantics of the flag. See + [#389](https://github.com/BurntSushi/ripgrep/issues/389) for more details. + +Feature enhancements: + +* [FEATURE #162](https://github.com/BurntSushi/ripgrep/issues/162): + libripgrep is now a thing. The primary crate is + [`grep`](https://docs.rs/grep). +* [FEATURE #176](https://github.com/BurntSushi/ripgrep/issues/176): + Add `-U/--multiline` flag that permits matching over multiple lines. +* [FEATURE #188](https://github.com/BurntSushi/ripgrep/issues/188): + Add `-P/--pcre2` flag that gives support for look-around and backreferences. +* [FEATURE #244](https://github.com/BurntSushi/ripgrep/issues/244): + Add `--json` flag that prints results in a JSON Lines format. +* [FEATURE #321](https://github.com/BurntSushi/ripgrep/issues/321): + Add `--one-file-system` flag to skip directories on different file systems. +* [FEATURE #404](https://github.com/BurntSushi/ripgrep/issues/404): + Add `--sort` and `--sortr` flag for more sorting. Deprecate `--sort-files`. +* [FEATURE #416](https://github.com/BurntSushi/ripgrep/issues/416): + Add `--crlf` flag to permit `$` to work with carriage returns on Windows. +* [FEATURE #917](https://github.com/BurntSushi/ripgrep/issues/917): + The `--trim` flag strips prefix whitespace from all lines printed. +* [FEATURE #993](https://github.com/BurntSushi/ripgrep/issues/993): + Add `--null-data` flag, which makes ripgrep use NUL as a line terminator. +* [FEATURE #997](https://github.com/BurntSushi/ripgrep/issues/997): + The `--passthru` flag now works with the `--replace` flag. +* [FEATURE #1038-1](https://github.com/BurntSushi/ripgrep/issues/1038): + Add `--line-buffered` and `--block-buffered` for forcing a buffer strategy. +* [FEATURE #1038-2](https://github.com/BurntSushi/ripgrep/issues/1038): + Add `--pre-glob` for filtering files through the `--pre` flag. + +Bug fixes: + +* [BUG #2](https://github.com/BurntSushi/ripgrep/issues/2): + Searching with non-zero context can now use memory maps if appropriate. +* [BUG #200](https://github.com/BurntSushi/ripgrep/issues/200): + ripgrep will now stop correctly when its output pipe is closed. +* [BUG #389](https://github.com/BurntSushi/ripgrep/issues/389): + The `-w/--word-regexp` flag now works more intuitively. +* [BUG #643](https://github.com/BurntSushi/ripgrep/issues/643): + Detection of readable stdin has improved on Windows. +* [BUG #441](https://github.com/BurntSushi/ripgrep/issues/441), + [BUG #690](https://github.com/BurntSushi/ripgrep/issues/690), + [BUG #980](https://github.com/BurntSushi/ripgrep/issues/980): + Matching empty lines now works correctly in several corner cases. +* [BUG #764](https://github.com/BurntSushi/ripgrep/issues/764): + Color escape sequences now coalesce, which reduces output size. +* [BUG #842](https://github.com/BurntSushi/ripgrep/issues/842): + Add man page to binary Debian package. +* [BUG #922](https://github.com/BurntSushi/ripgrep/issues/922): + ripgrep is now more robust with respect to memory maps failing. +* [BUG #937](https://github.com/BurntSushi/ripgrep/issues/937): + Color escape sequences are no longer emitted for empty matches. +* [BUG #940](https://github.com/BurntSushi/ripgrep/issues/940): + Context from the `--passthru` flag should not impact process exit status. +* [BUG #984](https://github.com/BurntSushi/ripgrep/issues/984): + Fixes bug in `ignore` crate where first path was always treated as a symlink. +* [BUG #990](https://github.com/BurntSushi/ripgrep/issues/990): + Read stderr asynchronously when running a process. +* [BUG #1013](https://github.com/BurntSushi/ripgrep/issues/1013): + Add compile time and runtime CPU features to `--version` output. +* [BUG #1028](https://github.com/BurntSushi/ripgrep/pull/1028): + Don't complete bare pattern after `-f` in zsh. + + +0.9.0 (2018-08-03) +================== +This is a new minor version release of ripgrep that contains some minor new +features and a panoply of bug fixes. + +Releases provided on Github for `x86_64` will now work on all target CPUs, and +will also automatically take advantage of features found on modern CPUs (such +as AVX2) for additional optimizations. + +This release increases the **minimum supported Rust version** from 1.20.0 to +1.23.0. + +It is anticipated that the next release of ripgrep (0.10.0) will provide +multi-line search support and a JSON output format. + +**BREAKING CHANGES**: + +* When `--count` and `--only-matching` are provided simultaneously, the + behavior of ripgrep is as if the `--count-matches` flag was given. That is, + the total number of matches is reported, where there may be multiple matches + per line. Previously, the behavior of ripgrep was to report the total number + of matching lines. (Note that this behavior diverges from the behavior of + GNU grep.) +* Octal syntax is no longer supported. ripgrep previously accepted expressions + like `\1` as syntax for matching `U+0001`, but ripgrep will now report an + error instead. +* The `--line-number-width` flag has been removed. Its functionality was not + carefully considered with all ripgrep output formats. + See [#795](https://github.com/BurntSushi/ripgrep/issues/795) for more + details. + +Feature enhancements: + +* Added or improved file type filtering for Android, Bazel, Fuchsia, Haskell, + Java and Puppet. +* [FEATURE #411](https://github.com/BurntSushi/ripgrep/issues/411): + Add a `--stats` flag, which emits aggregate statistics after search results. +* [FEATURE #646](https://github.com/BurntSushi/ripgrep/issues/646): + Add a `--no-ignore-messages` flag, which suppresses parse errors from reading + `.ignore` and `.gitignore` files. +* [FEATURE #702](https://github.com/BurntSushi/ripgrep/issues/702): + Support `\u{..}` Unicode escape sequences. +* [FEATURE #812](https://github.com/BurntSushi/ripgrep/issues/812): + Add `-b/--byte-offset` flag that shows the byte offset of each matching line. +* [FEATURE #814](https://github.com/BurntSushi/ripgrep/issues/814): + Add `--count-matches` flag, which is like `--count`, but for each match. +* [FEATURE #880](https://github.com/BurntSushi/ripgrep/issues/880): + Add a `--no-column` flag, which disables column numbers in the output. +* [FEATURE #898](https://github.com/BurntSushi/ripgrep/issues/898): + Add support for `lz4` when using the `-z/--search-zip` flag. +* [FEATURE #924](https://github.com/BurntSushi/ripgrep/issues/924): + `termcolor` has moved to its own repository: + https://github.com/BurntSushi/termcolor +* [FEATURE #934](https://github.com/BurntSushi/ripgrep/issues/934): + Add a new flag, `--no-ignore-global`, that permits disabling global + gitignores. +* [FEATURE #967](https://github.com/BurntSushi/ripgrep/issues/967): + Rename `--maxdepth` to `--max-depth` for consistency. Keep `--maxdepth` for + backwards compatibility. +* [FEATURE #978](https://github.com/BurntSushi/ripgrep/issues/978): + Add a `--pre` option to filter inputs with an arbitrary program. +* [FEATURE fca9709d](https://github.com/BurntSushi/ripgrep/commit/fca9709d): + Improve zsh completion. + +Bug fixes: + +* [BUG #135](https://github.com/BurntSushi/ripgrep/issues/135): + Release portable binaries that conditionally use SSSE3, AVX2, etc., at + runtime. +* [BUG #268](https://github.com/BurntSushi/ripgrep/issues/268): + Print descriptive error message when trying to use look-around or + backreferences. +* [BUG #395](https://github.com/BurntSushi/ripgrep/issues/395): + Show comprehensible error messages for regexes like `\s*{`. +* [BUG #526](https://github.com/BurntSushi/ripgrep/issues/526): + Support backslash escapes in globs. +* [BUG #795](https://github.com/BurntSushi/ripgrep/issues/795): + Fix problems with `--line-number-width` by removing it. +* [BUG #832](https://github.com/BurntSushi/ripgrep/issues/832): + Clarify usage instructions for `-f/--file` flag. +* [BUG #835](https://github.com/BurntSushi/ripgrep/issues/835): + Fix small performance regression while crawling very large directory trees. +* [BUG #851](https://github.com/BurntSushi/ripgrep/issues/851): + Fix `-S/--smart-case` detection once and for all. +* [BUG #852](https://github.com/BurntSushi/ripgrep/issues/852): + Be robust with respect to `ENOMEM` errors returned by `mmap`. +* [BUG #853](https://github.com/BurntSushi/ripgrep/issues/853): + Upgrade `grep` crate to `regex-syntax 0.6.0`. +* [BUG #893](https://github.com/BurntSushi/ripgrep/issues/893): + Improve support for git submodules. +* [BUG #900](https://github.com/BurntSushi/ripgrep/issues/900): + When no patterns are given, ripgrep should never match anything. +* [BUG #907](https://github.com/BurntSushi/ripgrep/issues/907): + ripgrep will now stop traversing after the first file when `--quiet --files` + is used. +* [BUG #918](https://github.com/BurntSushi/ripgrep/issues/918): + Don't skip tar archives when `-z/--search-zip` is used. +* [BUG #934](https://github.com/BurntSushi/ripgrep/issues/934): + Don't respect gitignore files when searching outside git repositories. +* [BUG #948](https://github.com/BurntSushi/ripgrep/issues/948): + Use exit code 2 to indicate error, and use exit code 1 to indicate no + matches. +* [BUG #951](https://github.com/BurntSushi/ripgrep/issues/951): + Add stdin example to ripgrep usage documentation. +* [BUG #955](https://github.com/BurntSushi/ripgrep/issues/955): + Use buffered writing when not printing to a tty, which fixes a performance + regression. +* [BUG #957](https://github.com/BurntSushi/ripgrep/issues/957): + Improve the error message shown for `--path separator /` in some Windows + shells. +* [BUG #964](https://github.com/BurntSushi/ripgrep/issues/964): + Add a `--no-fixed-strings` flag to disable `-F/--fixed-strings`. +* [BUG #988](https://github.com/BurntSushi/ripgrep/issues/988): + Fix a bug in the `ignore` crate that prevented the use of explicit ignore + files after disabling all other ignore rules. +* [BUG #995](https://github.com/BurntSushi/ripgrep/issues/995): + Respect `$XDG_CONFIG_DIR/git/config` for detecting `core.excludesFile`. + + +0.8.1 (2018-02-20) +================== +This is a patch release of ripgrep that primarily fixes regressions introduced +in 0.8.0 (#820 and #824) in directory traversal on Windows. These regressions +do not impact non-Windows users. + +Feature enhancements: + +* Added or improved file type filtering for csv and VHDL. +* [FEATURE #798](https://github.com/BurntSushi/ripgrep/issues/798): + Add `underline` support to `termcolor` and ripgrep. See documentation on the + `--colors` flag for details. + +Bug fixes: + +* [BUG #684](https://github.com/BurntSushi/ripgrep/issues/684): + Improve documentation for the `--ignore-file` flag. +* [BUG #789](https://github.com/BurntSushi/ripgrep/issues/789): + Don't show `(rev )` if the revision wasn't available during the build. +* [BUG #791](https://github.com/BurntSushi/ripgrep/issues/791): + Add man page to ARM release. +* [BUG #797](https://github.com/BurntSushi/ripgrep/issues/797): + Improve documentation for "intense" setting in `termcolor`. +* [BUG #800](https://github.com/BurntSushi/ripgrep/issues/800): + Fix a bug in the `ignore` crate for custom ignore files. This had no impact + on ripgrep. +* [BUG #807](https://github.com/BurntSushi/ripgrep/issues/807): + Fix a bug where `rg --hidden .` behaved differently from `rg --hidden ./`. +* [BUG #815](https://github.com/BurntSushi/ripgrep/issues/815): + Clarify a common failure mode in user guide. +* [BUG #820](https://github.com/BurntSushi/ripgrep/issues/820): + Fixes a bug on Windows where symlinks were followed even if not requested. +* [BUG #824](https://github.com/BurntSushi/ripgrep/issues/824): + Fix a performance regression in directory traversal on Windows. + + +0.8.0 (2018-02-11) +================== +This is a new minor version release of ripgrep that satisfies several popular +feature requests (config files, search compressed files, true colors), fixes +many bugs and improves the quality of life for ripgrep maintainers. This +release also includes greatly improved documentation in the form of a +[User Guide](GUIDE.md) and a [FAQ](FAQ.md). + +This release increases the **minimum supported Rust version** from 1.17 to +1.20. + +**BREAKING CHANGES**: + +Note that these are all very minor and unlikely to impact most users. + +* In order to support configuration files, flag overrides needed to be + rethought. In some cases, this changed ripgrep's behavior. For example, + in ripgrep 0.7.1, `rg foo -s -i` will perform a case sensitive search + since the `-s/--case-sensitive` flag was defined to always take precedence + over the `-i/--ignore-case` flag, regardless of position. In ripgrep 0.8.0 + however, the override rule for all flags has changed to "the most recent + flag wins among competing flags." That is, `rg foo -s -i` now performs a + case insensitive search. +* The `-M/--max-columns` flag was tweaked so that specifying a value of `0` + now makes ripgrep behave as if the flag was absent. This makes it possible + to set a default value in a configuration file and then override it. The + previous ripgrep behavior was to suppress all matching non-empty lines. +* In all globs, `[^...]` is now equivalent to `[!...]` (indicating class + negation). Previously, `^` had no special significance in a character class. +* For **downstream packagers**, the directory hierarchy in ripgrep's archive + releases has changed. The root directory now only contains the executable, + README and license. There is now a new directory called `doc` which contains + the man page (previously in the root), a user guide (new), a FAQ (new) and + the CHANGELOG (previously not included in release). The `complete` + directory remains the same. + +Feature enhancements: + +* Added or improved file type filtering for + Apache Avro, C++, GN, Google Closure Templates, Jupyter notebooks, man pages, + Protocol Buffers, Smarty and Web IDL. +* [FEATURE #196](https://github.com/BurntSushi/ripgrep/issues/196): + Support a configuration file. See + [the new user guide](GUIDE.md#configuration-file) + for details. +* [FEATURE #261](https://github.com/BurntSushi/ripgrep/issues/261): + Add extended or "true" color support. Works in Windows 10! + [See the FAQ for details.](FAQ.md#colors) +* [FEATURE #539](https://github.com/BurntSushi/ripgrep/issues/539): + Search gzip, bzip2, lzma or xz files when given `-z/--search-zip` flag. +* [FEATURE #544](https://github.com/BurntSushi/ripgrep/issues/544): + Add support for line number alignment via a new `--line-number-width` flag. +* [FEATURE #654](https://github.com/BurntSushi/ripgrep/pull/654): + Support linuxbrew in ripgrep's Brew tap. +* [FEATURE #673](https://github.com/BurntSushi/ripgrep/issues/673): + Bring back `.rgignore` files. (A higher precedent, application specific + version of `.ignore`.) +* [FEATURE #676](https://github.com/BurntSushi/ripgrep/issues/676): + Provide ARM binaries. **WARNING:** This will be provided on a best effort + basis. +* [FEATURE #709](https://github.com/BurntSushi/ripgrep/issues/709): + Suggest `-F/--fixed-strings` flag on a regex syntax error. +* [FEATURE #740](https://github.com/BurntSushi/ripgrep/issues/740): + Add a `--passthru` flag that causes ripgrep to print every line it reads. +* [FEATURE #785](https://github.com/BurntSushi/ripgrep/pull/785): + Overhaul documentation. Cleaned up README, added user guide and FAQ. +* [FEATURE 7f5c07](https://github.com/BurntSushi/ripgrep/commit/7f5c07434be92103b5bf7e216b9c7494aed2d8cb): + Add hidden flags for convenient overrides (e.g., `--no-text`). + +Bug fixes: + +* [BUG #553](https://github.com/BurntSushi/ripgrep/issues/553): + Permit flags to be repeated. +* [BUG #633](https://github.com/BurntSushi/ripgrep/issues/633): + Fix a bug where ripgrep would panic on Windows while following symlinks. +* [BUG #649](https://github.com/BurntSushi/ripgrep/issues/649): + Fix handling of `!**/` in `.gitignore`. +* [BUG #663](https://github.com/BurntSushi/ripgrep/issues/663): + **BREAKING CHANGE:** Support `[^...]` glob syntax (as identical to `[!...]`). +* [BUG #693](https://github.com/BurntSushi/ripgrep/issues/693): + Don't display context separators when not printing matches. +* [BUG #705](https://github.com/BurntSushi/ripgrep/issues/705): + Fix a bug that prevented ripgrep from searching OneDrive directories. +* [BUG #717](https://github.com/BurntSushi/ripgrep/issues/717): + Improve `--smart-case` uppercase character detection. +* [BUG #725](https://github.com/BurntSushi/ripgrep/issues/725): + Clarify that globs do not override explicitly given paths to search. +* [BUG #742](https://github.com/BurntSushi/ripgrep/pull/742): + Write ANSI reset code as `\x1B[0m` instead of `\x1B[m`. +* [BUG #747](https://github.com/BurntSushi/ripgrep/issues/747): + Remove `yarn.lock` from YAML file type. +* [BUG #760](https://github.com/BurntSushi/ripgrep/issues/760): + ripgrep can now search `/sys/devices/system/cpu/vulnerabilities/*` files. +* [BUG #761](https://github.com/BurntSushi/ripgrep/issues/761): + Fix handling of gitignore patterns that contain a `/`. +* [BUG #776](https://github.com/BurntSushi/ripgrep/pull/776): + **BREAKING CHANGE:** `--max-columns=0` now disables the limit. +* [BUG #779](https://github.com/BurntSushi/ripgrep/issues/779): + Clarify documentation for `--files-without-match`. +* [BUG #780](https://github.com/BurntSushi/ripgrep/issues/780), + [BUG #781](https://github.com/BurntSushi/ripgrep/issues/781): + Fix bug where ripgrep missed some matching lines. + +Maintenance fixes: + +* [MAINT #772](https://github.com/BurntSushi/ripgrep/pull/772): + Drop `env_logger` in favor of simpler logger to avoid many new dependencies. +* [MAINT #772](https://github.com/BurntSushi/ripgrep/pull/772): + Add git revision hash to ripgrep's version string. +* [MAINT #772](https://github.com/BurntSushi/ripgrep/pull/772): + (Seemingly) improve compile times. +* [MAINT #776](https://github.com/BurntSushi/ripgrep/pull/776): + Automatically generate man page during build. +* [MAINT #786](https://github.com/BurntSushi/ripgrep/pull/786): + Remove use of `unsafe` in `globset`. :tada: +* [MAINT e9d448](https://github.com/BurntSushi/ripgrep/commit/e9d448e93bb4e1fb3b0c1afc29adb5af6ed5283d): + Add an issue template (has already drastically improved bug reports). +* [MAINT ae2d03](https://github.com/BurntSushi/ripgrep/commit/ae2d036dd4ba2a46acac9c2d77c32e7c667eb850): + Remove the `compile` script. + +Friends of ripgrep: + +I'd like to extend my gratitude to +[@balajisivaraman](https://github.com/balajisivaraman) +for their recent hard work in a number of areas, and in particular, for +implementing the "search compressed files" feature. Their work in sketching out +a specification for that and other work has been exemplary. + +Thanks +[@balajisivaraman](https://github.com/balajisivaraman)! + + +0.7.1 (2017-10-22) +================== +This is a patch release of ripgrep that includes a fix to very bad regression +introduced in ripgrep 0.7.0. + +Bug fixes: + +* [BUG #648](https://github.com/BurntSushi/ripgrep/issues/648): + Fix a bug where it was very easy to exceed standard file descriptor limits. + + +0.7.0 (2017-10-20) +================== +This is a new minor version release of ripgrep that includes mostly bug fixes. + +ripgrep continues to require Rust 1.17, and there are no known breaking changes +introduced in this release. + +Feature enhancements: + +* Added or improved file type filtering for config & license files, Elm, + Purescript, Standard ML, sh, systemd, Terraform +* [FEATURE #593](https://github.com/BurntSushi/ripgrep/pull/593): + Using both `-o/--only-matching` and `-r/--replace` does the right thing. + +Bug fixes: + +* [BUG #200](https://github.com/BurntSushi/ripgrep/issues/200): + ripgrep will stop when its pipe is closed. +* [BUG #402](https://github.com/BurntSushi/ripgrep/issues/402): + Fix context printing bug when the `-m/--max-count` flag is used. +* [BUG #521](https://github.com/BurntSushi/ripgrep/issues/521): + Fix interaction between `-r/--replace` and terminal colors. +* [BUG #559](https://github.com/BurntSushi/ripgrep/issues/559): + Ignore test that tried reading a non-UTF-8 file path on macOS. +* [BUG #599](https://github.com/BurntSushi/ripgrep/issues/599): + Fix color escapes on empty matches. +* [BUG #600](https://github.com/BurntSushi/ripgrep/issues/600): + Avoid expensive (on Windows) file handle check when using --files. +* [BUG #618](https://github.com/BurntSushi/ripgrep/issues/618): + Clarify installation instructions for Ubuntu users. +* [BUG #633](https://github.com/BurntSushi/ripgrep/issues/633): + Faster symlink loop checking on Windows. + + +0.6.0 (2017-08-23) +================== +This is a new minor version release of ripgrep that includes many bug fixes +and a few new features such as `--iglob` and `-x/--line-regexp`. + +Note that this release increases the minimum supported Rust version from 1.12 +to 1.17. + +Feature enhancements: + +* Added or improved file type filtering for BitBake, C++, Cabal, cshtml, Julia, + Make, msbuild, QMake, Yocto +* [FEATURE #163](https://github.com/BurntSushi/ripgrep/issues/163): + Add an `--iglob` flag that is like `-g/--glob`, but matches globs + case insensitively. +* [FEATURE #520](https://github.com/BurntSushi/ripgrep/pull/518): + Add `-x/--line-regexp` flag, which requires a match to span an entire line. +* [FEATURE #551](https://github.com/BurntSushi/ripgrep/pull/551), + [FEATURE #554](https://github.com/BurntSushi/ripgrep/pull/554): + `ignore`: add new `matched_path_or_any_parents` method. + +Bug fixes: + +* [BUG #342](https://github.com/BurntSushi/ripgrep/issues/342): + Fix invisible text in some PowerShell environments by changing the + default color scheme on Windows. +* [BUG #413](https://github.com/BurntSushi/ripgrep/issues/413): + Release binaries on Unix are now `strip`'d by default. This decreases + binary size by an order of magnitude. +* [BUG #483](https://github.com/BurntSushi/ripgrep/issues/483): + When `--quiet` is passed, `--files` should be quiet. +* [BUG #488](https://github.com/BurntSushi/ripgrep/pull/488): + When `--vimgrep` is passed, `--with-filename` should be enabled + automatically. +* [BUG #493](https://github.com/BurntSushi/ripgrep/issues/493): + Fix another bug in the implementation of the `-o/--only-matching` + flag. +* [BUG #499](https://github.com/BurntSushi/ripgrep/pull/499): + Permit certain flags to override others. +* [BUG #523](https://github.com/BurntSushi/ripgrep/pull/523): + `wincolor`: Re-fetch Windows console on all calls. +* [BUG #523](https://github.com/BurntSushi/ripgrep/issues/524): + `--version` now shows enabled compile-time features. +* [BUG #532](https://github.com/BurntSushi/ripgrep/issues/532), + [BUG #536](https://github.com/BurntSushi/ripgrep/pull/536), + [BUG #538](https://github.com/BurntSushi/ripgrep/pull/538), + [BUG #540](https://github.com/BurntSushi/ripgrep/pull/540), + [BUG #560](https://github.com/BurntSushi/ripgrep/pull/560), + [BUG #565](https://github.com/BurntSushi/ripgrep/pull/565): + Improve zsh completion. +* [BUG #578](https://github.com/BurntSushi/ripgrep/pull/578): + Enable SIMD for `encoding_rs` when appropriate. +* [BUG #580](https://github.com/BurntSushi/ripgrep/issues/580): + Fix `-w/--word-regexp` in the presence of capturing groups. +* [BUG #581](https://github.com/BurntSushi/ripgrep/issues/581): + Document that ripgrep may terminate unexpectedly when searching via + memory maps (which can happen using default settings). + +Friends of ripgrep: + +I'd like to give a big Thank You to @okdana for their recent hard work on +ripgrep. This includes new features like `--line-regexp`, heroic effort on +zsh auto-completion and thinking through some thorny argv issues with me. + +I'd also like to thank @ericbn for their work on improving ripgrep's argv +parsing by allowing some flags to override others. + +Thanks @okdana and @ericbn! + + +0.5.2 (2017-05-11) +================== +Feature enhancements: + +* Added or improved file type filtering for Nix. +* [FEATURE #362](https://github.com/BurntSushi/ripgrep/issues/362): + Add `--regex-size-limit` and `--dfa-size-limit` flags. +* [FEATURE #444](https://github.com/BurntSushi/ripgrep/issues/444): + Improve error messages for invalid globs. + +Bug fixes: + +* [BUG #442](https://github.com/BurntSushi/ripgrep/issues/442): + Fix line wrapping in `--help` output. +* [BUG #451](https://github.com/BurntSushi/ripgrep/issues/451): + Fix bug with duplicate output when using `-o/--only-matching` flag. + + +0.5.1 (2017-04-09) +================== +Feature enhancements: + +* Added or improved file type filtering for vim. +* [FEATURE #34](https://github.com/BurntSushi/ripgrep/issues/34): + Add a `-o/--only-matching` flag. +* [FEATURE #377](https://github.com/BurntSushi/ripgrep/issues/377): + Column numbers can now be customized with a color. (The default is + no color.) +* [FEATURE #419](https://github.com/BurntSushi/ripgrep/issues/419): + Added `-0` short flag option for `--null`. + +Bug fixes: + +* [BUG #381](https://github.com/BurntSushi/ripgrep/issues/381): + Include license text in all subcrates. +* [BUG #418](https://github.com/BurntSushi/ripgrep/issues/418), + [BUG #426](https://github.com/BurntSushi/ripgrep/issues/426), + [BUG #439](https://github.com/BurntSushi/ripgrep/issues/439): + Fix a few bugs with `-h/--help` output. + + +0.5.0 (2017-03-12) +================== +This is a new minor version release of ripgrep that includes one minor breaking +change, bug fixes and several new features including support for text encodings +other than UTF-8. + +A notable accomplishment with respect to Rust is that ripgrep proper now only +contains a single `unsafe` use (for accessing the contents of a memory map). + +The **breaking change** is: + +* [FEATURE #380](https://github.com/BurntSushi/ripgrep/issues/380): + Line numbers are now hidden by default when ripgrep is printing to a tty + **and** the only thing searched is stdin. + +Feature enhancements: + +* Added or improved file type filtering for Ceylon, CSS, Elixir, HTML, log, + SASS, SVG, Twig +* [FEATURE #1](https://github.com/BurntSushi/ripgrep/issues/1): + Add support for additional text encodings, including automatic detection for + UTF-16 via BOM sniffing. Explicit text encoding support with the + `-E/--encoding` flag was also added for latin-1, GBK, EUC-JP + and Shift_JIS, among others. The full list can be found here: + https://encoding.spec.whatwg.org/#concept-encoding-get +* [FEATURE #129](https://github.com/BurntSushi/ripgrep/issues/129): + Add a new `-M/--max-columns` flag that omits lines longer than the given + number of bytes. (Disabled by default!) +* [FEATURE #369](https://github.com/BurntSushi/ripgrep/issues/369): + A new flag, `--max-filesize`, was added for limiting searches to files with + a maximum file size. + +Bug fixes: + +* [BUG #52](https://github.com/BurntSushi/ripgrep/issues/52), + [BUG #311](https://github.com/BurntSushi/ripgrep/issues/311): + Tweak how binary files are detected and handled. (We are slightly less + conservative and will no longer use memory without bound.) +* [BUG #326](https://github.com/BurntSushi/ripgrep/issues/326): + When --files flag is given, we should never attempt to parse positional + arguments as regexes. +* [BUG #327](https://github.com/BurntSushi/ripgrep/issues/327): + Permit the --heading flag to override the --no-heading flag. +* [BUG #340](https://github.com/BurntSushi/ripgrep/pull/340): + Clarify that the `-u/--unrestricted` flags are aliases. +* [BUG #343](https://github.com/BurntSushi/ripgrep/pull/343): + Global git ignore config should use `$HOME/.config/git/ignore` and not + `$HOME/git/ignore`. +* [BUG #345](https://github.com/BurntSushi/ripgrep/pull/345): + Clarify docs for `-g/--glob` flag. +* [BUG #381](https://github.com/BurntSushi/ripgrep/issues/381): + Add license files to each sub-crate. +* [BUG #383](https://github.com/BurntSushi/ripgrep/issues/383): + Use latest version of clap (for argv parsing). +* [BUG #392](https://github.com/BurntSushi/ripgrep/issues/391): + Fix translation of set globs (e.g., `{foo,bar,quux}`) to regexes. +* [BUG #401](https://github.com/BurntSushi/ripgrep/pull/401): + Add PowerShell completion file to Windows release. +* [BUG #405](https://github.com/BurntSushi/ripgrep/issues/405): + Fix bug when excluding absolute paths with the `-g/--glob` flag. + + +0.4.0 +===== +This is a new minor version release of ripgrep that includes a couple very +minor breaking changes, a few new features and lots of bug fixes. + +This version of ripgrep upgrades its `regex` dependency from `0.1` to `0.2`, +which includes a few minor syntax changes: + +* POSIX character classes now require double bracketing. Previously, the regex + `[:upper:]` would parse as the `upper` POSIX character class. Now it parses + as the character class containing the characters `:upper:`. The fix to this + change is to use `[[:upper:]]` instead. Note that variants like + `[[:upper:][:blank:]]` continue to work. +* The character `[` must always be escaped inside a character class. +* The characters `&`, `-` and `~` must be escaped if any one of them are + repeated consecutively. For example, `[&]`, `[\&]`, `[\&\&]`, `[&-&]` are all + equivalent while `[&&]` is illegal. (The motivation for this and the prior + change is to provide a backwards compatible path for adding character class + set notation.) + +Feature enhancements: + +* Added or improved file type filtering for Crystal, Kotlin, Perl, PowerShell, + Ruby, Swig +* [FEATURE #83](https://github.com/BurntSushi/ripgrep/issues/83): + Type definitions can now include other type definitions. +* [FEATURE #243](https://github.com/BurntSushi/ripgrep/issues/243): + **BREAKING CHANGE**: The `--column` flag now implies `--line-number`. +* [FEATURE #263](https://github.com/BurntSushi/ripgrep/issues/263): + Add a new `--sort-files` flag. +* [FEATURE #275](https://github.com/BurntSushi/ripgrep/issues/275): + Add a new `--path-separator` flag. Useful in cygwin. + +Bug fixes: + +* [BUG #182](https://github.com/BurntSushi/ripgrep/issues/182): + Redux: use more portable ANSI color escape sequences when possible. +* [BUG #258](https://github.com/BurntSushi/ripgrep/issues/258): + Fix bug that caused ripgrep's parallel iterator to spin and burn CPU. +* [BUG #262](https://github.com/BurntSushi/ripgrep/issues/262): + Document how to install shell completion files. +* [BUG #266](https://github.com/BurntSushi/ripgrep/issues/266), + [BUG #293](https://github.com/BurntSushi/ripgrep/issues/293): + Fix handling of bold styling and change the default colors. +* [BUG #268](https://github.com/BurntSushi/ripgrep/issues/268): + Make lack of backreference support more explicit. +* [BUG #271](https://github.com/BurntSushi/ripgrep/issues/271): + Remove `~` dependency on clap. +* [BUG #277](https://github.com/BurntSushi/ripgrep/issues/277): + Fix cosmetic issue in `globset` crate docs. +* [BUG #279](https://github.com/BurntSushi/ripgrep/issues/279): + ripgrep did not terminate when `-q/--quiet` was given. +* [BUG #281](https://github.com/BurntSushi/ripgrep/issues/281): + **BREAKING CHANGE**: Completely remove `^C` handling from ripgrep. +* [BUG #284](https://github.com/BurntSushi/ripgrep/issues/284): + Make docs for `-g/--glob` clearer. +* [BUG #286](https://github.com/BurntSushi/ripgrep/pull/286): + When stdout is redirected to a file, don't search that file. +* [BUG #287](https://github.com/BurntSushi/ripgrep/pull/287): + Fix ZSH completions. +* [BUG #295](https://github.com/BurntSushi/ripgrep/pull/295): + Remove superfluous `memmap` dependency in `grep` crate. +* [BUG #308](https://github.com/BurntSushi/ripgrep/pull/308): + Improve docs for `-r/--replace`. +* [BUG #313](https://github.com/BurntSushi/ripgrep/pull/313): + Update bytecount dep to latest version. +* [BUG #318](https://github.com/BurntSushi/ripgrep/pull/318): + Fix invalid UTF-8 output bug in Windows consoles. + + +0.3.2 +===== +Feature enhancements: + +* Added or improved file type filtering for Less, Sass, stylus, Zsh + +Bug fixes: + +* [BUG #229](https://github.com/BurntSushi/ripgrep/issues/229): + Make smart case slightly less conservative. +* [BUG #247](https://github.com/BurntSushi/ripgrep/issues/247): + Clarify use of --heading/--no-heading. +* [BUG #251](https://github.com/BurntSushi/ripgrep/issues/251), + [BUG #264](https://github.com/BurntSushi/ripgrep/issues/264), + [BUG #267](https://github.com/BurntSushi/ripgrep/issues/267): + Fix matching bug caused by literal optimizations. +* [BUG #256](https://github.com/BurntSushi/ripgrep/issues/256): + Fix bug that caused `rg foo` and `rg foo/` to have different behavior + when `foo` was a symlink. +* [BUG #270](https://github.com/BurntSushi/ripgrep/issues/270): + Fix bug where patterns starting with a `-` couldn't be used with the + `-e/--regexp` flag. (This resolves a regression that was introduced in + ripgrep 0.3.0.) + + +0.3.1 +===== +Bug fixes: + +* [BUG #242](https://github.com/BurntSushi/ripgrep/issues/242): + ripgrep didn't respect `--colors foo:none` correctly. Now it does. + + +0.3.0 +===== +This is a new minor version release of ripgrep that includes two breaking +changes with lots of bug fixes and some new features and performance +improvements. Notably, if you had a problem with colors or piping on Windows +before, then that should now be fixed in this release. + +**BREAKING CHANGES**: + +* ripgrep now requires Rust 1.11 to compile. Previously, it could build on + Rust 1.9. The cause of this was the move from + [Docopt to Clap](https://github.com/BurntSushi/ripgrep/pull/233) + for argument parsing. +* The `-e/--regexp` flag can no longer accept a pattern starting with a `-`. + There are two work-arounds: `rg -- -foo` and `rg [-]foo` or `rg -e [-]foo` + will all search for the same `-foo` pattern. The cause of this was the move + from [Docopt to Clap](https://github.com/BurntSushi/ripgrep/pull/233) + for argument parsing. + [This may get fixed in the + future.](https://github.com/kbknapp/clap-rs/issues/742). + +Performance improvements: + +* [PERF #33](https://github.com/BurntSushi/ripgrep/issues/33): + ripgrep now performs similar to GNU grep on small corpora. +* [PERF #136](https://github.com/BurntSushi/ripgrep/issues/136): + ripgrep no longer slows down because of argument parsing when given a large + argument list. + +Feature enhancements: + +* Added or improved file type filtering for Elixir. +* [FEATURE #7](https://github.com/BurntSushi/ripgrep/issues/7): + Add a `-f/--file` flag that causes ripgrep to read patterns from a file. +* [FEATURE #51](https://github.com/BurntSushi/ripgrep/issues/51): + Add a `--colors` flag that enables one to customize the colors used in + ripgrep's output. +* [FEATURE #138](https://github.com/BurntSushi/ripgrep/issues/138): + Add a `--files-without-match` flag that shows only file paths that contain + zero matches. +* [FEATURE #230](https://github.com/BurntSushi/ripgrep/issues/230): + Add completion files to the release (Bash, Fish and PowerShell). + +Bug fixes: + +* [BUG #37](https://github.com/BurntSushi/ripgrep/issues/37): + Use correct ANSI escape sequences when `TERM=screen.linux`. +* [BUG #94](https://github.com/BurntSushi/ripgrep/issues/94): + ripgrep now detects stdin on Windows automatically. +* [BUG #117](https://github.com/BurntSushi/ripgrep/issues/117): + Colors should now work correctly and automatically inside mintty. +* [BUG #182](https://github.com/BurntSushi/ripgrep/issues/182): + Colors should now work within Emacs. In particular, `--color=always` will + emit colors regardless of the current environment. +* [BUG #189](https://github.com/BurntSushi/ripgrep/issues/189): + Show less content when running `rg -h`. The full help content can be + accessed with `rg --help`. +* [BUG #210](https://github.com/BurntSushi/ripgrep/issues/210): + Support non-UTF-8 file names on Unix platforms. +* [BUG #231](https://github.com/BurntSushi/ripgrep/issues/231): + Switch from block buffering to line buffering. +* [BUG #241](https://github.com/BurntSushi/ripgrep/issues/241): + Some error messages weren't suppressed when `--no-messages` was used. + + +0.2.9 +===== +Bug fixes: + +* [BUG #226](https://github.com/BurntSushi/ripgrep/issues/226): + File paths explicitly given on the command line weren't searched in parallel. + (This was a regression in `0.2.7`.) +* [BUG #228](https://github.com/BurntSushi/ripgrep/issues/228): + If a directory was given to `--ignore-file`, ripgrep's memory usage would + grow without bound. + + +0.2.8 +===== +Bug fixes: + +* Fixed a bug with the SIMD/AVX features for using bytecount in commit + `4ca15a`. + + +0.2.7 +===== +Performance improvements: + +* [PERF #223](https://github.com/BurntSushi/ripgrep/pull/223): + Added a parallel recursive directory iterator. This results in major + performance improvements on large repositories. +* [PERF #11](https://github.com/BurntSushi/ripgrep/pull/11): + ripgrep now uses the `bytecount` library for counting new lines. In some + cases, ripgrep runs twice as fast. Use + `RUSTFLAGS="-C target-cpu=native" cargo build --release --features 'simd-accel avx-accel'` + to get the fastest possible binary. + +Feature enhancements: + +* Added or improved file type filtering for Agda, Tex, Taskpaper, Markdown, + asciidoc, textile, rdoc, org, creole, wiki, pod, C#, PDF, C, C++. +* [FEATURE #149](https://github.com/BurntSushi/ripgrep/issues/149): + Add a new `--no-messages` flag that suppresses error messages. + Note that `rg foo 2> /dev/null` also works. +* [FEATURE #159](https://github.com/BurntSushi/ripgrep/issues/159): + Add a new `-m/--max-count` flag that limits the total number of matches + printed for each file searched. + +Bug fixes: + +* [BUG #199](https://github.com/BurntSushi/ripgrep/issues/199): + Fixed a bug where `-S/--smart-case` wasn't being applied correctly to + literal optimizations. +* [BUG #203](https://github.com/BurntSushi/ripgrep/issues/203): + Mention the full name, ripgrep, in more places. It now appears in + the output of `--help` and `--version`. The repository URL is now also + in the output of `--help` and the man page. +* [BUG #215](https://github.com/BurntSushi/ripgrep/issues/215): + Include small note about how to search for a pattern that starts with a `-`. + + +0.2.6 +===== +Feature enhancements: + +* Added or improved file type filtering for Fish. + +Bug fixes: + +* [BUG #206](https://github.com/BurntSushi/ripgrep/issues/206): + Fixed a regression with `-g/--glob` flag in `0.2.5`. + + +0.2.5 +===== +Feature enhancements: + +* Added or improved file type filtering for Groovy, Handlebars, Tcl, zsh and + Python. +* [FEATURE #9](https://github.com/BurntSushi/ripgrep/issues/9): + Support global gitignore config and `.git/info/exclude` files. +* [FEATURE #45](https://github.com/BurntSushi/ripgrep/issues/45): + Add --ignore-file flag for specifying additional ignore files. +* [FEATURE #202](https://github.com/BurntSushi/ripgrep/pull/202): + Introduce a new + [`ignore`](https://github.com/BurntSushi/ripgrep/tree/master/ignore) + crate that encapsulates all of ripgrep's gitignore matching logic. + +Bug fixes: + +* [BUG #44](https://github.com/BurntSushi/ripgrep/issues/44): + ripgrep runs slowly when given lots of positional arguments that are + directories. +* [BUG #119](https://github.com/BurntSushi/ripgrep/issues/119): + ripgrep didn't reset terminal colors if it was interrupted by `^C`. + Fixed in [PR #187](https://github.com/BurntSushi/ripgrep/pull/187). +* [BUG #184](https://github.com/BurntSushi/ripgrep/issues/184): + Fixed a bug related to interpreting gitignore files in parent directories. + + +0.2.4 +===== +SKIPPED. + + +0.2.3 +===== +Bug fixes: + +* [BUG #164](https://github.com/BurntSushi/ripgrep/issues/164): + Fixes a segfault on macos builds. +* [BUG #167](https://github.com/BurntSushi/ripgrep/issues/167): + Clarify documentation for --threads. + + +0.2.2 +===== +Packaging updates: + +* `ripgrep` is now in homebrew-core. `brew install ripgrep` will do the trick + on a Mac. +* `ripgrep` is now in the Archlinux community repository. + `pacman -S ripgrep` will do the trick on Archlinux. +* Support has been discontinued for i686-darwin. +* Glob matching has been moved out into its own crate: + [`globset`](https://crates.io/crates/globset). + +Feature enhancements: + +* Added or improved file type filtering for CMake, config, Jinja, Markdown, + Spark. +* [FEATURE #109](https://github.com/BurntSushi/ripgrep/issues/109): + Add a --max-depth flag for directory traversal. +* [FEATURE #124](https://github.com/BurntSushi/ripgrep/issues/124): + Add -s/--case-sensitive flag. Overrides --smart-case. +* [FEATURE #139](https://github.com/BurntSushi/ripgrep/pull/139): + The `ripgrep` repo is now a Homebrew tap. This is useful for installing + SIMD accelerated binaries, which aren't available in homebrew-core. + +Bug fixes: + +* [BUG #87](https://github.com/BurntSushi/ripgrep/issues/87), + [BUG #127](https://github.com/BurntSushi/ripgrep/issues/127), + [BUG #131](https://github.com/BurntSushi/ripgrep/issues/131): + Various issues related to glob matching. +* [BUG #116](https://github.com/BurntSushi/ripgrep/issues/116): + --quiet should stop search after first match. +* [BUG #121](https://github.com/BurntSushi/ripgrep/pull/121): + --color always should show colors, even when --vimgrep is used. +* [BUG #122](https://github.com/BurntSushi/ripgrep/pull/122): + Colorize file path at beginning of line. +* [BUG #134](https://github.com/BurntSushi/ripgrep/issues/134): + Processing a large ignore file (thousands of globs) was very slow. +* [BUG #137](https://github.com/BurntSushi/ripgrep/issues/137): + Always follow symlinks when given as an explicit argument. +* [BUG #147](https://github.com/BurntSushi/ripgrep/issues/147): + Clarify documentation for --replace. + + +0.2.1 +===== +Feature enhancements: + +* Added or improved file type filtering for Clojure and SystemVerilog. +* [FEATURE #89](https://github.com/BurntSushi/ripgrep/issues/89): + Add a --null flag that outputs a NUL byte after every file path. + +Bug fixes: + +* [BUG #98](https://github.com/BurntSushi/ripgrep/issues/98): + Fix a bug in single threaded mode when if opening a file failed, ripgrep + quit instead of continuing the search. +* [BUG #99](https://github.com/BurntSushi/ripgrep/issues/99): + Fix another bug in single threaded mode where empty lines were being printed + by mistake. +* [BUG #105](https://github.com/BurntSushi/ripgrep/issues/105): + Fix an off-by-one error with --column. +* [BUG #106](https://github.com/BurntSushi/ripgrep/issues/106): + Fix a bug where a whitespace only line in a gitignore file caused ripgrep + to panic (i.e., crash). + + +0.2.0 +===== +Feature enhancements: + +* Added or improved file type filtering for VB, R, F#, Swift, Nim, JavaScript, + TypeScript +* [FEATURE #20](https://github.com/BurntSushi/ripgrep/issues/20): + Adds a --no-filename flag. +* [FEATURE #26](https://github.com/BurntSushi/ripgrep/issues/26): + Adds --files-with-matches flag. Like --count, but only prints file paths + and doesn't need to count every match. +* [FEATURE #40](https://github.com/BurntSushi/ripgrep/issues/40): + Switch from using `.rgignore` to `.ignore`. Note that `.rgignore` is + still supported, but deprecated. +* [FEATURE #68](https://github.com/BurntSushi/ripgrep/issues/68): + Add --no-ignore-vcs flag that ignores .gitignore but not .ignore. +* [FEATURE #70](https://github.com/BurntSushi/ripgrep/issues/70): + Add -S/--smart-case flag (but is disabled by default). +* [FEATURE #80](https://github.com/BurntSushi/ripgrep/issues/80): + Add support for `{foo,bar}` globs. + +Many many bug fixes. Thanks every for reporting these and helping make +`ripgrep` better! (Note that I haven't captured every tracking issue here, +some were closed as duplicates.) + +* [BUG #8](https://github.com/BurntSushi/ripgrep/issues/8): + Don't use an intermediate buffer when --threads=1. (Permits constant memory + usage.) +* [BUG #15](https://github.com/BurntSushi/ripgrep/issues/15): + Improves the documentation for --type-add. +* [BUG #16](https://github.com/BurntSushi/ripgrep/issues/16), + [BUG #49](https://github.com/BurntSushi/ripgrep/issues/49), + [BUG #50](https://github.com/BurntSushi/ripgrep/issues/50), + [BUG #65](https://github.com/BurntSushi/ripgrep/issues/65): + Some gitignore globs were being treated as anchored when they weren't. +* [BUG #18](https://github.com/BurntSushi/ripgrep/issues/18): + --vimgrep reported incorrect column number. +* [BUG #19](https://github.com/BurntSushi/ripgrep/issues/19): + ripgrep was hanging waiting on stdin in some Windows terminals. Note that + this introduced a new bug: + [#94](https://github.com/BurntSushi/ripgrep/issues/94). +* [BUG #21](https://github.com/BurntSushi/ripgrep/issues/21): + Removes leading `./` when printing file paths. +* [BUG #22](https://github.com/BurntSushi/ripgrep/issues/22): + Running `rg --help | echo` caused `rg` to panic. +* [BUG #24](https://github.com/BurntSushi/ripgrep/issues/22): + Clarify the central purpose of rg in its usage message. +* [BUG #25](https://github.com/BurntSushi/ripgrep/issues/25): + Anchored gitignore globs weren't applied in subdirectories correctly. +* [BUG #30](https://github.com/BurntSushi/ripgrep/issues/30): + Globs like `foo/**` should match contents of `foo`, but not `foo` itself. +* [BUG #35](https://github.com/BurntSushi/ripgrep/issues/35), + [BUG #81](https://github.com/BurntSushi/ripgrep/issues/81): + When automatically detecting stdin, only read if it's a file or a fifo. + i.e., ignore stdin in `rg foo < /dev/null`. +* [BUG #36](https://github.com/BurntSushi/ripgrep/issues/36): + Don't automatically pick memory maps on MacOS. Ever. +* [BUG #38](https://github.com/BurntSushi/ripgrep/issues/38): + Trailing whitespace in gitignore wasn't being ignored. +* [BUG #43](https://github.com/BurntSushi/ripgrep/issues/43): + --glob didn't work with directories. +* [BUG #46](https://github.com/BurntSushi/ripgrep/issues/46): + Use one fewer worker thread than what is provided on CLI. +* [BUG #47](https://github.com/BurntSushi/ripgrep/issues/47): + --help/--version now work even if other options are set. +* [BUG #55](https://github.com/BurntSushi/ripgrep/issues/55): + ripgrep was refusing to search /proc/cpuinfo. Fixed by disabling memory + maps for files with zero size. +* [BUG #64](https://github.com/BurntSushi/ripgrep/issues/64): + The first path given with --files set was ignored. +* [BUG #67](https://github.com/BurntSushi/ripgrep/issues/67): + Sometimes whitelist globs like `!/dir` weren't interpreted as anchored. +* [BUG #77](https://github.com/BurntSushi/ripgrep/issues/77): + When -q/--quiet flag was passed, ripgrep kept searching even after a match + was found. +* [BUG #90](https://github.com/BurntSushi/ripgrep/issues/90): + Permit whitelisting hidden files. +* [BUG #93](https://github.com/BurntSushi/ripgrep/issues/93): + ripgrep was extracting an erroneous inner literal from a repeated pattern. diff --git a/userland/ripgrep/CONTRIBUTING.md b/userland/ripgrep/CONTRIBUTING.md new file mode 100644 index 0000000..a7eaba9 --- /dev/null +++ b/userland/ripgrep/CONTRIBUTING.md @@ -0,0 +1,8 @@ +# Contributing + +## Use of AI + +All use of AI in contributions must follow the +[AI Policy](https://github.com/BurntSushi/ripgrep/blob/master/AI_POLICY.md). + +Contributions not following the AI Policy will be closed. diff --git a/userland/ripgrep/COPYING b/userland/ripgrep/COPYING new file mode 100644 index 0000000..bb9c20a --- /dev/null +++ b/userland/ripgrep/COPYING @@ -0,0 +1,3 @@ +This project is dual-licensed under the Unlicense and MIT licenses. + +You may use this code under the terms of either license. diff --git a/userland/ripgrep/Cargo.lock b/userland/ripgrep/Cargo.lock new file mode 100644 index 0000000..ba9d392 --- /dev/null +++ b/userland/ripgrep/Cargo.lock @@ -0,0 +1,513 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "aho-corasick" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +dependencies = [ + "memchr", +] + +[[package]] +name = "anyhow" +version = "1.0.103" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a4385e2e34eb35d6b3efe798b9eb88096925d87726c0798709bf56d9ed84af3" + +[[package]] +name = "bstr" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f7dc094d718f2e1c1559ad110e27eeaae14a5465d3d56dd6dbd793079fbd530" +dependencies = [ + "memchr", + "regex-automata", + "serde_core", +] + +[[package]] +name = "cc" +version = "1.2.67" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e17dd265a7d0f31ef544e1b20e03add05d3b45b491b633b10d67145d2acc1a38" +dependencies = [ + "find-msvc-tools", + "jobserver", + "libc", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "crossbeam-deque" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5181e0de7b61eb03a81e347d6dd8797bae9da5146707b51077e2d71a54ec0ceb" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17" + +[[package]] +name = "encoding_rs" +version = "0.8.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "encoding_rs_io" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1cc3c5651fb62ab8aa3103998dade57efdd028544bd300516baa31840c252a83" +dependencies = [ + "encoding_rs", +] + +[[package]] +name = "find-msvc-tools" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "libc", + "r-efi", +] + +[[package]] +name = "globset" +version = "0.4.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e47d37d2ae4464254884b60ab7071be2b876a9c35b696bd018ddcc76847309cd" +dependencies = [ + "aho-corasick", + "bstr", + "log", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "grep" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "309217bc53e2c691c314389c7fa91f9cd1a998cda19e25544ea47d94103880c3" +dependencies = [ + "grep-cli", + "grep-matcher", + "grep-pcre2", + "grep-printer", + "grep-regex", + "grep-searcher", +] + +[[package]] +name = "grep-cli" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf32d263c5d5cc2a23ce587097f5ddafdb188492ba2e6fb638eaccdc22453631" +dependencies = [ + "bstr", + "globset", + "libc", + "log", + "termcolor", + "winapi-util", +] + +[[package]] +name = "grep-matcher" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9417543f4870fc8f1c8e1af870afae2431007626d9e703fce6471c468d33847" +dependencies = [ + "memchr", +] + +[[package]] +name = "grep-pcre2" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b752bb6f57dd3716e55b63a33ca05b5825c3d66cd427468874a5564af6ca365c" +dependencies = [ + "grep-matcher", + "log", + "pcre2", +] + +[[package]] +name = "grep-printer" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd76035e87871f51c1ee5b793e32122b3ccf9c692662d9622ef1686ff5321acb" +dependencies = [ + "bstr", + "grep-matcher", + "grep-searcher", + "log", + "serde", + "serde_json", + "termcolor", +] + +[[package]] +name = "grep-regex" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ce0c256c3ad82bcc07b812c15a45ec1d398122e8e15124f96695234db7112ef" +dependencies = [ + "bstr", + "grep-matcher", + "log", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "grep-searcher" +version = "0.1.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72348823a0eafc4bc2e9051064f28b5b42cc100b571b3a35d67918d711efcbc6" +dependencies = [ + "bstr", + "encoding_rs", + "encoding_rs_io", + "grep-matcher", + "log", + "memchr", + "memmap2", +] + +[[package]] +name = "ignore" +version = "0.4.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4ffa3a0547a138e59ddd6fa3b7c672ed47e6ad6a3cd177984ff1116aa5ba742" +dependencies = [ + "crossbeam-deque", + "globset", + "log", + "memchr", + "regex-automata", + "same-file", + "walkdir", + "winapi-util", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "jobserver" +version = "0.1.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c00acbd29eabad4a2392fa0e921c874934dbbf4194312ad20f04a0ed67a3cb3" +dependencies = [ + "getrandom", + "libc", +] + +[[package]] +name = "lexopt" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "803ec87c9cfb29b9d2633f20cba1f488db3fd53f2158b1024cbefb47ba05d413" + +[[package]] +name = "libc" +version = "0.2.186" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" + +[[package]] +name = "log" +version = "0.4.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "memmap2" +version = "0.9.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1219ed1b7f229ee7104d281dd01d6802fe28bb6e95d292942c4daacdeb798c0" +dependencies = [ + "libc", +] + +[[package]] +name = "pcre2" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e970b0fcce0c7ee6ef662744ff711f21ccd6f11b7cf03cd187a80e89797fc67" +dependencies = [ + "libc", + "log", + "pcre2-sys", +] + +[[package]] +name = "pcre2-sys" +version = "0.2.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18b9073c1a2549bd409bf4a32c94d903bb1a09bf845bc306ae148897fa0760a4" +dependencies = [ + "cc", + "libc", + "pkg-config", +] + +[[package]] +name = "pkg-config" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" + +[[package]] +name = "proc-macro2" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfbc457d0c7a0759a614551b11a6409e5951f6c7537be1f1b7682b9ae9230368" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "regex-automata" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f388202e4b80542a0921078cc23b6333bcf1409c1e3f86404cae4766a6131db" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "ripgrep" +version = "15.2.0" +dependencies = [ + "anyhow", + "bstr", + "grep", + "ignore", + "lexopt", + "log", + "serde", + "serde_derive", + "serde_json", + "termcolor", + "textwrap", + "tikv-jemallocator", + "walkdir", +] + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.150" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "termcolor" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06794f8f6c5c898b3275aebefa6b8a1cb24cd2c6c79397ab15774837a0bc5755" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "textwrap" +version = "0.16.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c13547615a44dc9c452a8a534638acdf07120d4b6847c8178705da06306a3057" + +[[package]] +name = "tikv-jemalloc-sys" +version = "0.7.1+5.3.1-0-g81034ce1f1373e37dc865038e1bc8eeecf559ce8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a2825c78386b4ae0314074867860ba9577875de945f05992c38815cbec327f0" +dependencies = [ + "cc", + "libc", +] + +[[package]] +name = "tikv-jemallocator" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "249f09e49ab1609436f34c776e84231bead18d6a955f119f939bdc1d847561bd" +dependencies = [ + "libc", + "tikv-jemalloc-sys", +] + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/userland/ripgrep/Cargo.toml b/userland/ripgrep/Cargo.toml new file mode 100644 index 0000000..9806ac1 --- /dev/null +++ b/userland/ripgrep/Cargo.toml @@ -0,0 +1,189 @@ +# THIS FILE IS AUTOMATICALLY GENERATED BY CARGO +# +# When uploading crates to the registry Cargo will automatically +# "normalize" Cargo.toml files for maximal compatibility +# with all versions of Cargo and also rewrite `path` dependencies +# to registry (e.g., crates.io) dependencies. +# +# If you are reading this file be aware that the original Cargo.toml +# will likely look very different (and much more reasonable). +# See Cargo.toml.orig for the original contents. + +[package] +edition = "2024" +rust-version = "1.85" +name = "ripgrep" +version = "15.2.0" +authors = ["Andrew Gallant "] +build = "build.rs" +exclude = [ + "HomebrewFormula", + "/.github/", + "/ci/", + "/pkg/brew", + "/benchsuite/", + "/scripts/", + "/crates/fuzz", +] +autolib = false +autobins = false +autoexamples = false +autotests = false +autobenches = false +description = """ +ripgrep is a line-oriented search tool that recursively searches the current +directory for a regex pattern while respecting gitignore rules. ripgrep has +first class support on Windows, macOS and Linux. +""" +homepage = "https://github.com/BurntSushi/ripgrep" +documentation = "https://github.com/BurntSushi/ripgrep" +readme = "README.md" +keywords = [ + "regex", + "grep", + "egrep", + "search", + "pattern", +] +categories = [ + "command-line-utilities", + "text-processing", +] +license = "Unlicense OR MIT" +repository = "https://github.com/BurntSushi/ripgrep" + +[package.metadata.deb] +features = ["pcre2"] +section = "utils" +assets = [ + [ + "target/release/rg", + "usr/bin/", + "755", +], + [ + "COPYING", + "usr/share/doc/ripgrep/", + "644", +], + [ + "LICENSE-MIT", + "usr/share/doc/ripgrep/", + "644", +], + [ + "UNLICENSE", + "usr/share/doc/ripgrep/", + "644", +], + [ + "CHANGELOG.md", + "usr/share/doc/ripgrep/CHANGELOG", + "644", +], + [ + "README.md", + "usr/share/doc/ripgrep/README", + "644", +], + [ + "FAQ.md", + "usr/share/doc/ripgrep/FAQ", + "644", +], + [ + "deployment/deb/rg.1", + "usr/share/man/man1/rg.1", + "644", +], + [ + "deployment/deb/rg.bash", + "usr/share/bash-completion/completions/rg", + "644", +], + [ + "deployment/deb/rg.fish", + "usr/share/fish/vendor_completions.d/rg.fish", + "644", +], + [ + "deployment/deb/_rg", + "usr/share/zsh/vendor-completions/", + "644", +], +] +extended-description = """ +ripgrep (rg) recursively searches your current directory for a regex pattern. +By default, ripgrep will respect your .gitignore and automatically skip hidden +files/directories and binary files. +""" + +[features] +pcre2 = ["grep/pcre2"] + +[[bin]] +name = "rg" +path = "crates/core/main.rs" +bench = false + +[[test]] +name = "integration" +path = "tests/tests.rs" + +[dependencies.anyhow] +version = "1.0.75" + +[dependencies.bstr] +version = "1.7.0" + +[dependencies.grep] +version = "0.4.1" + +[dependencies.ignore] +version = "0.4.29" + +[dependencies.lexopt] +version = "0.3.0" + +[dependencies.log] +version = "0.4.5" + +[dependencies.serde_json] +version = "1.0.23" + +[dependencies.termcolor] +version = "1.4.0" + +[dependencies.textwrap] +version = "0.16.0" +default-features = false + +[dev-dependencies.serde] +version = "1.0.77" + +[dev-dependencies.serde_derive] +version = "1.0.77" + +[dev-dependencies.walkdir] +version = "2" + +[target.'cfg(all(target_env = "musl", target_pointer_width = "64"))'.dependencies.tikv-jemallocator] +version = "0.7.0" + +[profile.deb] +inherits = "release-lto" + +[profile.release] +debug = 1 + +[profile.release-lto] +opt-level = 3 +lto = "fat" +codegen-units = 1 +debug = 0 +debug-assertions = false +panic = "abort" +overflow-checks = false +incremental = false +inherits = "release" +strip = "symbols" diff --git a/userland/ripgrep/Cargo.toml.orig b/userland/ripgrep/Cargo.toml.orig new file mode 100644 index 0000000..0fce841 --- /dev/null +++ b/userland/ripgrep/Cargo.toml.orig @@ -0,0 +1,117 @@ +[package] +name = "ripgrep" +version = "15.2.0" #:version +authors = ["Andrew Gallant "] +description = """ +ripgrep is a line-oriented search tool that recursively searches the current +directory for a regex pattern while respecting gitignore rules. ripgrep has +first class support on Windows, macOS and Linux. +""" +documentation = "https://github.com/BurntSushi/ripgrep" +homepage = "https://github.com/BurntSushi/ripgrep" +repository = "https://github.com/BurntSushi/ripgrep" +keywords = ["regex", "grep", "egrep", "search", "pattern"] +categories = ["command-line-utilities", "text-processing"] +license = "Unlicense OR MIT" +exclude = [ + "HomebrewFormula", + "/.github/", + "/ci/", + "/pkg/brew", + "/benchsuite/", + "/scripts/", + "/crates/fuzz", +] +build = "build.rs" +autotests = false +edition = "2024" +rust-version = "1.85" + +[[bin]] +bench = false +path = "crates/core/main.rs" +name = "rg" + +[[test]] +name = "integration" +path = "tests/tests.rs" + +[workspace] +members = [ + "crates/globset", + "crates/grep", + "crates/cli", + "crates/matcher", + "crates/pcre2", + "crates/printer", + "crates/regex", + "crates/searcher", + "crates/ignore", +] + +[dependencies] +anyhow = "1.0.75" +bstr = "1.7.0" +grep = { version = "0.4.1", path = "crates/grep" } +ignore = { version = "0.4.29", path = "crates/ignore" } +lexopt = "0.3.0" +log = "0.4.5" +serde_json = "1.0.23" +termcolor = "1.4.0" +textwrap = { version = "0.16.0", default-features = false } + +[target.'cfg(all(target_env = "musl", target_pointer_width = "64"))'.dependencies.tikv-jemallocator] +version = "0.7.0" + +[dev-dependencies] +serde = "1.0.77" +serde_derive = "1.0.77" +walkdir = "2" + +[features] +pcre2 = ["grep/pcre2"] + +[profile.release] +debug = 1 + +[profile.release-lto] +inherits = "release" +opt-level = 3 +debug = "none" +strip = "symbols" +debug-assertions = false +overflow-checks = false +lto = "fat" +panic = "abort" +incremental = false +codegen-units = 1 + +[profile.deb] +inherits = "release-lto" + +[package.metadata.deb] +features = ["pcre2"] +section = "utils" +assets = [ + ["target/release/rg", "usr/bin/", "755"], + ["COPYING", "usr/share/doc/ripgrep/", "644"], + ["LICENSE-MIT", "usr/share/doc/ripgrep/", "644"], + ["UNLICENSE", "usr/share/doc/ripgrep/", "644"], + ["CHANGELOG.md", "usr/share/doc/ripgrep/CHANGELOG", "644"], + ["README.md", "usr/share/doc/ripgrep/README", "644"], + ["FAQ.md", "usr/share/doc/ripgrep/FAQ", "644"], + # The man page is automatically generated by ripgrep's build process, so + # this file isn't actually committed. Instead, to create a dpkg, either + # create a deployment/deb directory and copy the man page to it, or use the + # 'ci/build-deb' script. + ["deployment/deb/rg.1", "usr/share/man/man1/rg.1", "644"], + # Similarly for shell completions. + ["deployment/deb/rg.bash", "usr/share/bash-completion/completions/rg", "644"], + ["deployment/deb/rg.fish", "usr/share/fish/vendor_completions.d/rg.fish", "644"], + ["deployment/deb/_rg", "usr/share/zsh/vendor-completions/", "644"], +] +extended-description = """\ +ripgrep (rg) recursively searches your current directory for a regex pattern. +By default, ripgrep will respect your .gitignore and automatically skip hidden +files/directories and binary files. +""" diff --git a/userland/ripgrep/FAQ.md b/userland/ripgrep/FAQ.md new file mode 100644 index 0000000..0ca3760 --- /dev/null +++ b/userland/ripgrep/FAQ.md @@ -0,0 +1,1063 @@ +## FAQ + +* [Does ripgrep support configuration files?](#config) +* [What's changed in ripgrep recently?](#changelog) +* [When is the next release?](#release) +* [Does ripgrep have a man page?](#manpage) +* [Does ripgrep have support for shell auto-completion?](#complete) +* [How can I get results in a consistent order?](#order) +* [How do I search files that aren't UTF-8?](#encoding) +* [How do I search compressed files?](#compressed) +* [How do I search over multiple lines?](#multiline) +* [How do I use lookaround and/or backreferences?](#fancy) +* [How do I configure ripgrep's colors?](#colors) +* [How do I enable true colors on Windows?](#truecolors-windows) +* [How do I stop ripgrep from messing up colors when I kill it?](#stop-ripgrep) +* [Why does using a leading `/` on Windows fail?](#because-cygwin) +* [How do I get around the regex size limit?](#size-limit) +* [How do I make the `-f/--file` flag faster?](#dfa-size) +* [How do I make the output look like The Silver Searcher's output?](#silver-searcher-output) +* [Why does ripgrep get slower when I enabled PCRE2 regexes?](#pcre2-slow) +* [When I run `rg`, why does it execute some other command?](#rg-other-cmd) +* [How do I create an alias for ripgrep on Windows?](#rg-alias-windows) +* [How do I create a PowerShell profile?](#powershell-profile) +* [How do I pipe non-ASCII content to ripgrep on Windows?](#pipe-non-ascii-windows) +* [How can I search and replace with ripgrep?](#search-and-replace) +* [How is ripgrep licensed?](#license) +* [Can ripgrep replace grep?](#posix4ever) +* [What does the "rip" in ripgrep mean?](#intentcountsforsomething) +* [How can I donate to ripgrep or its maintainers?](#donations) + + +

+Does ripgrep support configuration files? +

+ +Yes. See the +[guide's section on configuration files](GUIDE.md#configuration-file). + + +

+What's changed in ripgrep recently? +

+ +Please consult ripgrep's [CHANGELOG](CHANGELOG.md). + + +

+When is the next release? +

+ +ripgrep is a project whose contributors are volunteers. A release schedule +adds undue stress to said volunteers. Therefore, releases are made on a best +effort basis and no dates **will ever be given**. + +An exception to this _can be_ high impact bugs. If a ripgrep release contains +a significant regression, then there will generally be a strong push to get a +patch release out with a fix. However, no promises are made. + + +

+Does ripgrep have a man page? +

+ +Yes. If you installed ripgrep through a package manager on a Unix system, then +it would have ideally been installed for you in the proper location. In which +case, `man rg` should just work. + +Otherwise, you can ask ripgrep to generate the man page: + +``` +$ mkdir -p man/man1 +$ rg --generate man > man/man1/rg.1 +$ MANPATH="$PWD/man" man rg +``` + +Or, if your version of `man` supports the `-l/--local-file` flag, then this +will suffice: + +``` +$ rg --generate man | man -l - +``` + +Note that the man page's documentation for options is equivalent to the output +shown in `rg --help`. To see more condensed documentation (one line per flag), +run `rg -h`. + +The man page is also included in all +[ripgrep binary releases](https://github.com/BurntSushi/ripgrep/releases). + + +

+Does ripgrep have support for shell auto-completion? +

+ +Yes! If you installed ripgrep through a package manager on a Unix system, then +the shell completion files included in the release archive should have been +installed for you automatically. If not, you can generate completions using +ripgrep's command line interface. + +For **bash**: + +``` +$ dir="$XDG_CONFIG_HOME/bash_completion" +$ mkdir -p "$dir" +$ rg --generate complete-bash > "$dir/rg.bash" +``` + +For **fish**: + +``` +$ dir="$XDG_CONFIG_HOME/fish/completions" +$ mkdir -p "$dir" +$ rg --generate complete-fish > "$dir/rg.fish" +``` + +For **zsh**, the recommended approach is: + +```zsh +$ dir="$HOME/.zsh-complete" +$ mkdir -p "$dir" +$ rg --generate complete-zsh > "$dir/_rg" +``` + +And then add `$HOME/.zsh-complete` to your `fpath` in, e.g., your +`$HOME/.zshrc` file: + +```zsh +fpath=($HOME/.zsh-complete $fpath) +``` + +Or if you'd prefer to load and generate completions at the same time, you can +add the following to your `$HOME/.zshrc` file: + +```zsh +$ source <(rg --generate complete-zsh) +``` + +Note though that while this approach is easier to setup, is generally slower +than the previous method, and will add more time to loading your shell prompt. + +For **PowerShell**, create the completions: + +``` +$ rg --generate complete-powershell > _rg.ps1 +``` + +And then add `. _rg.ps1` to your PowerShell +[profile](https://technet.microsoft.com/en-us/library/bb613488(v=vs.85).aspx) +(note the leading period). If the `_rg.ps1` file is not on your `PATH`, do +`. /path/to/_rg.ps1` instead. + + +

+How can I get results in a consistent order? +

+ +By default, ripgrep uses parallelism to execute its search because this makes +the search much faster on most modern systems. This in turn means that ripgrep +has a non-deterministic aspect to it, since the interleaving of threads during +the execution of the program is itself non-deterministic. This has the effect +of printing results in a somewhat arbitrary order, and this order can change +from run to run of ripgrep. + +The only way to make the order of results consistent is to ask ripgrep to +sort the output. Currently, this will disable all parallelism. (On smaller +repositories, you might not notice much of a performance difference!) You +can achieve this with the `--sort path` flag. + +There is more discussion on this topic here: +https://github.com/BurntSushi/ripgrep/issues/152 + + +

+How do I search files that aren't UTF-8? +

+ +See the [guide's section on file encoding](GUIDE.md#file-encoding). + + +

+How do I search compressed files? +

+ +ripgrep's `-z/--search-zip` flag will cause it to search compressed files +automatically. Currently, this supports gzip, bzip2, xz, lzma, lz4, Brotli and +Zstd. Each of these requires the corresponding `gzip`, `bzip2`, `xz`, +`lz4`, `brotli` and `zstd` binaries to be installed on your system. (That is, +ripgrep does decompression by shelling out to another process.) + +ripgrep currently does not search archive formats, so `*.tar.gz` files, for +example, are skipped. + + +

+How do I search over multiple lines? +

+ +The `-U/--multiline` flag enables ripgrep to report results that span over +multiple lines. + + +

+How do I use lookaround and/or backreferences? +

+ +ripgrep's default regex engine does not support lookaround or backreferences. +This is primarily because the default regex engine is implemented using finite +state machines in order to guarantee a linear worst case time complexity on all +inputs. Backreferences are not possible to implement in this paradigm, and +lookaround appears difficult to do efficiently. + +However, ripgrep optionally supports using PCRE2 as the regex engine instead of +the default one based on finite state machines. You can enable PCRE2 with the +`-P/--pcre2` flag. For example, in the root of the ripgrep repo, you can easily +find all palindromes: + +``` +$ rg -P '(\w{10})\1' +tests/misc.rs +483: cmd.arg("--max-filesize").arg("44444444444444444444"); +globset/src/glob.rs +1206: matches!(match7, "a*a*a*a*a*a*a*a*a", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"); +``` + +If your version of ripgrep doesn't support PCRE2, then you'll get an error +message when you try to use the `-P/--pcre2` flag: + +``` +$ rg -P '(\w{10})\1' +PCRE2 is not available in this build of ripgrep +``` + +Most of the releases distributed by the ripgrep project here on GitHub will +come bundled with PCRE2 enabled. If you installed ripgrep through a different +means (like your system's package manager), then please reach out to the +maintainer of that package to see whether it's possible to enable the PCRE2 +feature. + + +

+How do I configure ripgrep's colors? +

+ +ripgrep has two flags related to colors: + +* `--color` controls *when* to use colors. +* `--colors` controls *which* colors to use. + +The `--color` flag accepts one of the following possible values: `never`, +`auto`, `always` or `ansi`. The `auto` value is the default and will cause +ripgrep to only enable colors when it is printing to a terminal. But if you +pipe ripgrep to a file or some other process, then it will suppress colors. + +The `--colors` flag is a bit more complicated. The general format is: + +``` +--colors '{type}:{attribute}:{value}' +``` + +* `{type}` should be one of `path`, `line`, `column` or `match`. Each of these + correspond to the four different types of things that ripgrep will add color + to in its output. Select the type whose color you want to change. +* `{attribute}` should be one of `fg`, `bg` or `style`, corresponding to + foreground color, background color, or miscellaneous styling (such as whether + to bold the output or not). +* `{value}` is determined by the value of `{attribute}`. If + `{attribute}` is `style`, then `{value}` should be one of `nobold`, + `bold`, `nointense`, `intense`, `nounderline`, `underline`, `noitalic` or + `italic`. If `{attribute}` is `fg` or `bg`, then `{value}` should be a color. + +A color is specified by either one of eight of English names, a single 256-bit +number or an RGB triple (with over 16 million possible values, or "true +color"). + +The color names are `red`, `blue`, `green`, `cyan`, `magenta`, `yellow`, +`white` or `black`. + +A single 256-bit number is a value in the range 0-255 (inclusive). It can +either be in decimal format (e.g., `62`) or hexadecimal format (e.g., `0x3E`). + +An RGB triple corresponds to three numbers (decimal or hexadecimal) separated +by commas. + +As a special case, `--colors '{type}:none'` will clear all colors and styles +associated with `{type}`, which lets you start with a clean slate (instead of +building on top of ripgrep's default color settings). + +Here's an example that highlights the matches with a nice blue background with +bolded white text: + +``` +$ rg somepattern \ + --colors 'match:none' \ + --colors 'match:bg:0x33,0x66,0xFF' \ + --colors 'match:fg:white' \ + --colors 'match:style:bold' +``` + +Colors are an ideal candidate to set in your +[configuration file](GUIDE.md#configuration-file). See the +[question on emulating The Silver Searcher's output style](#silver-searcher-output) +for an example specific to colors. + + +

+How do I enable true colors on Windows? +

+ +First, see the previous question's +[answer on configuring colors](#colors). + +Secondly, coloring on Windows is a bit complicated. If you're using a terminal +like Cygwin, then it's likely true color support already works out of the box. +However, if you are using a normal Windows console (`cmd` or `PowerShell`) and +a version of Windows prior to 10, then there is no known way to get true +color support. If you are on Windows 10 and using a Windows console, then +true colors should work out of the box with one caveat: you might need to +clear ripgrep's default color settings first. That is, instead of this: + +``` +$ rg somepattern --colors 'match:fg:0x33,0x66,0xFF' +``` + +you should do this + +``` +$ rg somepattern --colors 'match:none' --colors 'match:fg:0x33,0x66,0xFF' +``` + +This is because ripgrep might set the default style for `match` to `bold`, and +it seems like Windows 10's VT100 support doesn't permit bold and true color +ANSI escapes to be used simultaneously. The work-around above will clear +ripgrep's default styling, allowing you to craft it exactly as desired. + + +

+How do I stop ripgrep from messing up colors when I kill it? +

+ +Type in `color` in cmd.exe (Command Prompt) and `echo -ne "\033[0m"` on +Unix-like systems to restore your original foreground color. + +In PowerShell, you can add the following code to your profile which will +restore the original foreground color when `Reset-ForegroundColor` is called. +Including the `Set-Alias` line will allow you to call it with simply `color`. + +```powershell +$OrigFgColor = $Host.UI.RawUI.ForegroundColor +function Reset-ForegroundColor { + $Host.UI.RawUI.ForegroundColor = $OrigFgColor +} +Set-Alias -Name color -Value Reset-ForegroundColor +``` + +PR [#187](https://github.com/BurntSushi/ripgrep/pull/187) fixed this, and it +was later deprecated in +[#281](https://github.com/BurntSushi/ripgrep/issues/281). A full explanation is +available +[here](https://github.com/BurntSushi/ripgrep/issues/281#issuecomment-269093893). + + +

+Why does using a leading `/` on Windows fail? +

+ +If you're using cygwin on Windows and try to search for a pattern beginning +with a `/`, then it's possible that cygwin is mangling that pattern without +your knowledge. For example, if you tried running `rg /foo` in a cygwin shell +on Windows, then cygwin might mistakenly perform path translation on `/foo`, +which would result in `rg C:/msys64/foo` being searched instead. + +You can fix this in one of three ways: + +1. Stop using cygwin. +2. Escape the leading slash with an additional slash. e.g., `rg //foo`. +3. Temporarily disable path translation by setting `MSYS_NO_PATHCONV=1`. e.g., + `MSYS_NO_PATHCONV=1 rg /foo`. + +For more details, see https://github.com/BurntSushi/ripgrep/issues/1277 + + +

+How do I get around the regex size limit? +

+ +If you've given ripgrep a particularly large pattern (or a large number of +smaller patterns), then it is possible that it will fail to compile because it +hit a pre-set limit. For example: + +``` +$ rg '\pL{1000}' +Compiled regex exceeds size limit of 10485760 bytes. +``` + +(Note: `\pL{1000}` may look small, but `\pL` is the character class containing +all Unicode letters, which is quite large. *And* it's repeated 1000 times.) + +In this case, you can work around by simply increasing the limit: + +``` +$ rg '\pL{1000}' --regex-size-limit 1G +``` + +Increasing the limit to 1GB does not necessarily mean that ripgrep will use +that much memory. The limit just says that it's allowed to (approximately) use +that much memory for constructing the regular expression. + + +

+How do I make the -f/--file flag faster? +

+ +The `-f/--file` permits one to give a file to ripgrep which contains a pattern +on each line. ripgrep will then report any line that matches any of the +patterns. + +If this pattern file gets too big, then it is possible ripgrep will slow down +dramatically. *Typically* this is because an internal cache is too small, and +will cause ripgrep to spill over to a slower but more robust regular expression +engine. If this is indeed the problem, then it is possible to increase this +cache and regain speed. The cache can be controlled via the `--dfa-size-limit` +flag. For example, using `--dfa-size-limit 1G` will set the cache size to 1GB. +(Note that this doesn't mean ripgrep will use 1GB of memory automatically, but +it will allow the regex engine to if it needs to.) + + +

+How do I make the output look like The Silver Searcher's output? +

+ +Use the `--colors` flag, like so: + +``` +rg --colors line:fg:yellow \ + --colors line:style:bold \ + --colors path:fg:green \ + --colors path:style:bold \ + --colors match:fg:black \ + --colors match:bg:yellow \ + --colors match:style:nobold \ + foo +``` + +Alternatively, add your color configuration to your ripgrep config file (which +is activated by setting the `RIPGREP_CONFIG_PATH` environment variable to point +to your config file). For example: + +``` +$ cat $HOME/.config/ripgrep/rc +--colors=line:fg:yellow +--colors=line:style:bold +--colors=path:fg:green +--colors=path:style:bold +--colors=match:fg:black +--colors=match:bg:yellow +--colors=match:style:nobold +$ RIPGREP_CONFIG_PATH=$HOME/.config/ripgrep/rc rg foo +``` + + +

+Why does ripgrep get slower when I enable PCRE2 regexes? +

+ +When you use the `--pcre2` (`-P` for short) flag, ripgrep will use the PCRE2 +regex engine instead of the default. Both regex engines are quite fast, +but PCRE2 provides a number of additional features such as look-around and +backreferences that many enjoy using. This is largely because PCRE2 uses +a backtracking implementation where as the default regex engine uses a finite +automaton based implementation. The former provides the ability to add lots of +bells and whistles over the latter, but the latter executes with worst case +linear time complexity. + +With that out of the way, if you've used `-P` with ripgrep, you may have +noticed that it can be slower. The reasons for why this is are quite complex, +and they are complex because the optimizations that ripgrep uses to implement +fast search are complex. + +The task ripgrep has before it is somewhat simple; all it needs to do is search +a file for occurrences of some pattern and then print the lines containing +those occurrences. The problem lies in what is considered a valid match and how +exactly we read the bytes from a file. + +In terms of what is considered a valid match, remember that ripgrep will only +report matches spanning a single line by default. The problem here is that +some patterns can match across multiple lines, and ripgrep needs to prevent +that from happening. For example, `foo\sbar` will match `foo\nbar`. The most +obvious way to achieve this is to read the data from a file, and then apply +the pattern search to that data for each line. The problem with this approach +is that it can be quite slow; it would be much faster to let the pattern +search across as much data as possible. It's faster because it gets rid of the +overhead of finding the boundaries of every line, and also because it gets rid +of the overhead of starting and stopping the pattern search for every single +line. (This is operating under the general assumption that matching lines are +much rarer than non-matching lines.) + +It turns out that we can use the faster approach by applying a very simple +restriction to the pattern: *statically prevent* the pattern from matching +through a `\n` character. Namely, when given a pattern like `foo\sbar`, +ripgrep will remove `\n` from the `\s` character class automatically. In some +cases, a simple removal is not so easy. For example, ripgrep will return an +error when your pattern includes a `\n` literal: + +``` +$ rg '\n' +the literal '"\n"' is not allowed in a regex +``` + +So what does this have to do with PCRE2? Well, ripgrep's default regex engine +exposes APIs for doing syntactic analysis on the pattern in a way that makes +it quite easy to strip `\n` from the pattern (or otherwise detect it and report +an error if stripping isn't possible). PCRE2 seemingly does not provide a +similar API, so ripgrep does not do any stripping when PCRE2 is enabled. This +forces ripgrep to use the "slow" search strategy of searching each line +individually. + +OK, so if enabling PCRE2 slows down the default method of searching because it +forces matches to be limited to a single line, then why is PCRE2 also sometimes +slower when performing multiline searches? Well, that's because there are +*multiple* reasons why using PCRE2 in ripgrep can be slower than the default +regex engine. This time, blame PCRE2's Unicode support, which ripgrep enables +by default. In particular, PCRE2 cannot simultaneously enable Unicode support +and search arbitrary data. That is, when PCRE2's Unicode support is enabled, +the data **must** be valid UTF-8 (to do otherwise is to invoke undefined +behavior). This is in contrast to ripgrep's default regex engine, which can +enable Unicode support and still search arbitrary data. ripgrep's default +regex engine simply won't match invalid UTF-8 for a pattern that can otherwise +only match valid UTF-8. Why doesn't PCRE2 do the same? This author isn't +familiar with its internals, so we can't comment on it here. + +The bottom line here is that we can't enable PCRE2's Unicode support without +simultaneously incurring a performance penalty for ensuring that we are +searching valid UTF-8. In particular, ripgrep will transcode the contents +of each file to UTF-8 while replacing invalid UTF-8 data with the Unicode +replacement codepoint. ripgrep then disables PCRE2's own internal UTF-8 +checking, since we've guaranteed the data we hand it will be valid UTF-8. The +reason why ripgrep takes this approach is because if we do hand PCRE2 invalid +UTF-8, then it will report a match error if it comes across an invalid UTF-8 +sequence. This is not good news for ripgrep, since it will stop it from +searching the rest of the file, and will also print potentially undesirable +error messages to users. + +All right, the above is a lot of information to swallow if you aren't already +familiar with ripgrep internals. Let's make this concrete with some examples. +First, let's get some data big enough to magnify the performance differences: + +``` +$ curl -O 'https://burntsushi.net/stuff/subtitles2016-sample.gz' +$ gzip -d subtitles2016-sample +$ md5sum subtitles2016-sample +e3cb796a20bbc602fbfd6bb43bda45f5 subtitles2016-sample +``` + +To search this data, we will use the pattern `^\w{42}$`, which contains exactly +one hit in the file and has no literals. Having no literals is important, +because it ensures that the regex engine won't use literal optimizations to +speed up the search. In other words, it lets us reason coherently about the +actual task that the regex engine is performing. + +Let's now walk through a few examples in light of the information above. First, +let's consider the default search using ripgrep's default regex engine and +then the same search with PCRE2: + +``` +$ time rg '^\w{42}$' subtitles2016-sample +21225780:EverymajordevelopmentinthehistoryofAmerica + +real 0m1.783s +user 0m1.731s +sys 0m0.051s + +$ time rg -P '^\w{42}$' subtitles2016-sample +21225780:EverymajordevelopmentinthehistoryofAmerica + +real 0m2.458s +user 0m2.419s +sys 0m0.038s +``` + +In this particular example, both pattern searches are using a Unicode aware +`\w` character class and both are counting lines in order to report line +numbers. The key difference here is that the first search will not search +line by line, but the second one will. We can observe which strategy ripgrep +uses by passing the `--trace` flag: + +``` +$ rg '^\w{42}$' subtitles2016-sample --trace +[... snip ...] +TRACE|grep_searcher::searcher|grep-searcher/src/searcher/mod.rs:622: Some("subtitles2016-sample"): searching via memory map +TRACE|grep_searcher::searcher|grep-searcher/src/searcher/mod.rs:712: slice reader: searching via slice-by-line strategy +TRACE|grep_searcher::searcher::core|grep-searcher/src/searcher/core.rs:61: searcher core: will use fast line searcher +[... snip ...] + +$ rg -P '^\w{42}$' subtitles2016-sample --trace +[... snip ...] +TRACE|grep_searcher::searcher|grep-searcher/src/searcher/mod.rs:622: Some("subtitles2016-sample"): searching via memory map +TRACE|grep_searcher::searcher|grep-searcher/src/searcher/mod.rs:705: slice reader: needs transcoding, using generic reader +TRACE|grep_searcher::searcher|grep-searcher/src/searcher/mod.rs:685: generic reader: searching via roll buffer strategy +TRACE|grep_searcher::searcher::core|grep-searcher/src/searcher/core.rs:63: searcher core: will use slow line searcher +[... snip ...] +``` + +The first says it is using the "fast line searcher" where as the latter says +it is using the "slow line searcher." The latter also shows that we are +decoding the contents of the file, which also impacts performance. + +Interestingly, in this case, the pattern does not match a `\n` and the file +we're searching is valid UTF-8, so neither the slow line-by-line search +strategy nor the decoding are necessary. We could fix the former issue with +better PCRE2 introspection APIs. We can actually fix the latter issue with +ripgrep's `--no-encoding` flag, which prevents the automatic UTF-8 decoding, +but will enable PCRE2's own UTF-8 validity checking. Unfortunately, it's slower +in my build of ripgrep: + +``` +$ time rg -P '^\w{42}$' subtitles2016-sample --no-encoding +21225780:EverymajordevelopmentinthehistoryofAmerica + +real 0m3.074s +user 0m3.021s +sys 0m0.051s +``` + +(Tip: use the `--trace` flag to verify that no decoding in ripgrep is +happening.) + +A possible reason why PCRE2's UTF-8 checking is slower is because it might +not be better than the highly optimized UTF-8 checking routines found in the +[`encoding_rs`](https://github.com/hsivonen/encoding_rs) library, which is what +ripgrep uses for UTF-8 decoding. Moreover, my build of ripgrep enables +`encoding_rs`'s SIMD optimizations, which may be in play here. + +Also, note that using the `--no-encoding` flag can cause PCRE2 to report +invalid UTF-8 errors, which causes ripgrep to stop searching the file: + +``` +$ cat invalid-utf8 +foobar + +$ xxd invalid-utf8 +00000000: 666f 6fff 6261 720a foo.bar. + +$ rg foo invalid-utf8 +1:foobar + +$ rg -P foo invalid-utf8 +1:foo�bar + +$ rg -P foo invalid-utf8 --no-encoding +invalid-utf8: PCRE2: error matching: UTF-8 error: illegal byte (0xfe or 0xff) +``` + +All right, so at this point, you might think that we could remove the penalty +for line-by-line searching by enabling multiline search. After all, our +particular pattern can't match across multiple lines anyway, so we'll still get +the results we want. Let's try it: + +``` +$ time rg -U '^\w{42}$' subtitles2016-sample +21225780:EverymajordevelopmentinthehistoryofAmerica + +real 0m1.803s +user 0m1.748s +sys 0m0.054s + +$ time rg -P -U '^\w{42}$' subtitles2016-sample +21225780:EverymajordevelopmentinthehistoryofAmerica + +real 0m2.962s +user 0m2.246s +sys 0m0.713s +``` + +Search times remain the same with the default regex engine, but the PCRE2 +search gets _slower_. What happened? The secrets can be revealed with the +`--trace` flag once again. In the former case, ripgrep actually detects that +the pattern can't match across multiple lines, and so will fall back to the +"fast line search" strategy as with our search without `-U`. + +However, for PCRE2, things are much worse. Namely, since Unicode mode is still +enabled, ripgrep is still going to decode UTF-8 to ensure that it hands only +valid UTF-8 to PCRE2. Unfortunately, one key downside of multiline search is +that ripgrep cannot do it incrementally. Since matches can be arbitrarily long, +ripgrep actually needs the entire file in memory at once. Normally, we can use +a memory map for this, but because we need to UTF-8 decode the file before +searching it, ripgrep winds up reading the entire contents of the file on to +the heap before executing a search. Owch. + +OK, so Unicode is killing us here. The file we're searching is _mostly_ ASCII, +so maybe we're OK with missing some data. (Try `rg '[\w--\p{ascii}]'` to see +non-ASCII word characters that an ASCII-only `\w` character class would miss.) +We can disable Unicode in both searches, but this is done differently depending +on the regex engine we use: + +``` +$ time rg '(?-u)^\w{42}$' subtitles2016-sample +21225780:EverymajordevelopmentinthehistoryofAmerica + +real 0m1.714s +user 0m1.669s +sys 0m0.044s + +$ time rg -P '^\w{42}$' subtitles2016-sample --no-pcre2-unicode +21225780:EverymajordevelopmentinthehistoryofAmerica + +real 0m1.997s +user 0m1.958s +sys 0m0.037s +``` + +For the most part, ripgrep's default regex engine performs about the same. +PCRE2 does improve a little bit, and is now almost as fast as the default +regex engine. If you look at the output of `--trace`, you'll see that ripgrep +will no longer perform UTF-8 decoding, but it does still use the slow +line-by-line searcher. + +At this point, we can combine all of our insights above: let's try to get off +of the slow line-by-line searcher by enabling multiline mode, and let's stop +UTF-8 decoding by disabling Unicode support: + +``` +$ time rg -U '(?-u)^\w{42}$' subtitles2016-sample +21225780:EverymajordevelopmentinthehistoryofAmerica + +real 0m1.714s +user 0m1.655s +sys 0m0.058s + +$ time rg -P -U '^\w{42}$' subtitles2016-sample --no-pcre2-unicode +21225780:EverymajordevelopmentinthehistoryofAmerica + +real 0m1.121s +user 0m1.071s +sys 0m0.048s +``` + +Ah, there's PCRE2's JIT shining! ripgrep's default regex engine once again +remains about the same, but PCRE2 no longer needs to search line-by-line and it +no longer needs to do any kind of UTF-8 checks. This allows the file to get +memory mapped and passed right through PCRE2's JIT at impressive speeds. (As +a brief and interesting historical note, the configuration of "memory map + +multiline + no-Unicode" is exactly the configuration used by The Silver +Searcher. This analysis perhaps sheds some reasoning as to why that +configuration is useful!) + +In summary, if you want PCRE2 to go as fast as possible and you don't care +about Unicode and you don't care about matches possibly spanning across +multiple lines, then enable multiline mode with `-U` and disable PCRE2's +Unicode support with the `--no-pcre2-unicode` flag. + +Caveat emptor: This author is not a PCRE2 expert, so there may be APIs that can +improve performance that the author missed. Similarly, there may be alternative +designs for a searching tool that are more amenable to how PCRE2 works. + + +

+When I run rg, why does it execute some other command? +

+ +It's likely that you have a shell alias or even another tool called `rg` which +is interfering with ripgrep. Run `which rg` to see what it is. + +(Notably, the Rails plug-in for +[Oh My Zsh](https://github.com/robbyrussell/oh-my-zsh/wiki/Plugins#rails) sets +up an `rg` alias for `rails generate`.) + +Problems like this can be resolved in one of several ways: + +* If you're using the OMZ Rails plug-in, disable it by editing the `plugins` + array in your zsh configuration. +* Temporarily bypass an existing `rg` alias by calling ripgrep as + `command rg`, `\rg`, or `'rg'`. +* Temporarily bypass an existing alias or another tool named `rg` by calling + ripgrep by its full path (e.g., `/usr/bin/rg` or `/usr/local/bin/rg`). +* Permanently disable an existing `rg` alias by adding `unalias rg` to the + bottom of your shell configuration file (e.g., `.bash_profile` or `.zshrc`). +* Give ripgrep its own alias that doesn't conflict with other tools/aliases by + adding a line like the following to the bottom of your shell configuration + file: `alias ripgrep='command rg'`. + + +

+How do I create an alias for ripgrep on Windows? +

+ +Often you can find a need to make alias for commands you use a lot that set +certain flags. But PowerShell function aliases do not behave like your typical +linux shell alias. You always need to propagate arguments and `stdin` input. +But it cannot be done simply as +`function grep() { $input | rg.exe --hidden $args }` + +Use below example as reference to how setup alias in PowerShell. + +```powershell +function grep { + $count = @($input).Count + $input.Reset() + + if ($count) { + $input | rg.exe --hidden $args + } + else { + rg.exe --hidden $args + } +} +``` + +PowerShell special variables: + +* input - is powershell `stdin` object that allows you to access its content. +* args - is array of arguments passed to this function. + +This alias checks whether there is `stdin` input and propagates only if there +is some lines. Otherwise empty `$input` will make powershell to trigger `rg` to +search empty `stdin`. + + +

+How do I create a PowerShell profile? +

+ +To customize powershell on start-up, there is a special PowerShell script that +has to be created. In order to find its location, type `$profile`. +See +[Microsoft's documentation](https://technet.microsoft.com/en-us/library/bb613488(v=vs.85).aspx) +for more details. + +Any PowerShell code in this file gets evaluated at the start of console. This +way you can have own aliases to be created at start. + + +

+How do I pipe non-ASCII content to ripgrep on Windows? +

+ +When piping input into native executables in PowerShell, the encoding of the +input is controlled by the `$OutputEncoding` variable. By default, this is set +to US-ASCII, and any characters in the pipeline that don't have encodings in +US-ASCII are converted to `?` (question mark) characters. + +To change this setting, set `$OutputEncoding` to a different encoding, as +represented by a .NET encoding object. Some common examples are below. The +value of this variable is reset when PowerShell restarts, so to make this +change take effect every time PowerShell is started add a line setting the +variable into your PowerShell profile. + +Example `$OutputEncoding` settings: + +* UTF-8 without BOM: `$OutputEncoding = [System.Text.UTF8Encoding]::new()` +* The console's output encoding: + `$OutputEncoding = [System.Console]::OutputEncoding` + +If you continue to have encoding problems, you can also force the encoding +that the console will use for printing to UTF-8 with +`[System.Console]::OutputEncoding = [System.Text.Encoding]::UTF8`. This +will also reset when PowerShell is restarted, so you can add that line +to your profile as well if you want to make the setting permanent. + +

+How can I search and replace with ripgrep? +

+ +Using ripgrep alone, you can't. ripgrep is a search tool that will never +touch your files. However, the output of ripgrep can be piped to other tools +that do modify files on disk. See +[this issue](https://github.com/BurntSushi/ripgrep/issues/74) for more +information. + +sed is one such tool that can modify files on disk. sed can take a filename +and a substitution command to search and replace in the specified file. +Files containing matching patterns can be provided to sed using + +``` +rg foo --files-with-matches +``` + +The output of this command is a list of filenames that contain a match for +the `foo` pattern. + +This list can be piped into `xargs`, which will split the filenames from +standard input into arguments for the command following xargs. You can use this +combination to pipe a list of filenames into sed for replacement. For example: + +``` +rg foo --files-with-matches | xargs sed -i 's/foo/bar/g' +``` + +will replace all instances of 'foo' with 'bar' in the files in which +ripgrep finds the foo pattern. The `-i` flag to sed indicates that you are +editing files in place, and `s/foo/bar/g` says that you are performing a +**s**ubstitution of the pattern `foo` for `bar`, and that you are doing this +substitution **g**lobally (all occurrences of the pattern in each file). + +Note: the above command assumes that you are using GNU sed. If you are using +BSD sed (the default on macOS and FreeBSD) then you must modify the above +command to be the following: + +``` +rg foo --files-with-matches | xargs sed -i '' 's/foo/bar/g' +``` + +The `-i` flag in BSD sed requires a file extension to be given to make backups +for all modified files. Specifying the empty string prevents file backups from +being made. + +Finally, if any of your file paths contain whitespace in them, then you might +need to delimit your file paths with a NUL terminator. This requires telling +ripgrep to output NUL bytes between each path, and telling xargs to read paths +delimited by NUL bytes: + +``` +rg foo --files-with-matches -0 | xargs -0 sed -i 's/foo/bar/g' +``` + +To learn more about sed, see the sed manual +[here](https://www.gnu.org/software/sed/manual/sed.html). + +Additionally, Facebook has a tool called +[fastmod](https://github.com/facebookincubator/fastmod) +that uses some of the same libraries as ripgrep and might provide a more +ergonomic search-and-replace experience. + + +

+How is ripgrep licensed? +

+ +ripgrep is dual licensed under the +[Unlicense](https://unlicense.org/) +and MIT licenses. Specifically, you may use ripgrep under the terms of either +license. + +The reason why ripgrep is dual licensed this way is two-fold: + +1. I, as ripgrep's author, would like to participate in a small bit of + ideological activism by promoting the Unlicense's goal: to disclaim + copyright monopoly interest. +2. I, as ripgrep's author, would like as many people to use ripgrep as + possible. Since the Unlicense is not a proven or well known license, ripgrep + is also offered under the MIT license, which is ubiquitous and accepted by + almost everyone. + +More specifically, ripgrep and all its dependencies are compatible with this +licensing choice. In particular, ripgrep's dependencies (direct and transitive) +will always be limited to permissive licenses. That is, ripgrep will never +depend on code that is not permissively licensed. This means rejecting any +dependency that uses a copyleft license such as the GPL, LGPL, MPL or any of +the Creative Commons ShareAlike licenses. Whether the license is "weak" +copyleft or not does not matter; ripgrep will **not** depend on it. + + +

+Can ripgrep replace grep? +

+ +Yes and no. + +If, upon hearing that "ripgrep can replace grep," you *actually* hear, "ripgrep +can be used in every instance grep can be used, in exactly the same way, for +the same use cases, with exactly the same bug-for-bug behavior," then no, +ripgrep trivially *cannot* replace grep. Moreover, ripgrep will *never* replace +grep. + +If, upon hearing that "ripgrep can replace grep," you *actually* hear, "ripgrep +can replace grep in some cases and not in other use cases," then yes, that is +indeed true! + +Let's go over some of those use cases in favor of ripgrep. Some of these may +not apply to you. That's OK. There may be other use cases not listed here that +do apply to you. That's OK too. + +(For all claims related to performance in the following words, see my +[blog post](https://blog.burntsushi.net/ripgrep/) +introducing ripgrep.) + +* Are you frequently searching a repository of code? If so, ripgrep might be a + good choice since there's likely a good chunk of your repository that you + don't want to search. grep, can, of course, be made to filter files using + recursive search, and if you don't mind writing out the requisite `--exclude` + rules or writing wrapper scripts, then grep might be sufficient. (I'm not + kidding, I myself did this with grep for almost a decade before writing + ripgrep.) But if you instead enjoy having a search tool respect your + `.gitignore`, then ripgrep might be perfect for you! +* Are you frequently searching non-ASCII text that is UTF-8 encoded? One of + ripgrep's key features is that it can handle Unicode features in your + patterns in a way that tends to be faster than GNU grep. Unicode features + in ripgrep are enabled by default; there is no need to configure your locale + settings to use ripgrep properly because ripgrep doesn't respect your locale + settings. +* Do you need to search UTF-16 files and you don't want to bother explicitly + transcoding them? Great. ripgrep does this for you automatically. No need + to enable it. +* Do you need to search a large directory of large files? ripgrep uses + parallelism by default, which tends to make it faster than a standard + `grep -r` search. However, if you're OK writing the occasional + `find ./ -print0 | xargs -P8 -0 grep` command, then maybe grep is good + enough. + +Here are some cases where you might *not* want to use ripgrep. The same caveats +for the previous section apply. + +* Are you writing portable shell scripts intended to work in a variety of + environments? Great, probably not a good idea to use ripgrep! ripgrep has + nowhere near the ubiquity of grep, so if you do use ripgrep, you might need + to futz with the installation process more than you would with grep. +* Do you care about POSIX compatibility? If so, then you can't use ripgrep + because it never was, isn't and never will be POSIX compatible. +* Do you hate tools that try to do something smart? If so, ripgrep is all about + being smart, so you might prefer to just stick with grep. +* Is there a particular feature of grep you rely on that ripgrep either doesn't + have or never will have? If the former, file a bug report, maybe ripgrep can + do it! If the latter, well, then, just use grep. + + +

+What does the "rip" in ripgrep mean? +

+ +When I first started writing ripgrep, I called it `rep`, intending it to be a +shorter variant of `grep`. Soon after, I renamed it to `xrep` since `rep` +wasn't obvious enough of a name for my taste. And also because adding `x` to +anything always makes it better, right? + +Before ripgrep's first public release, I decided that I didn't like `xrep`. I +thought it was slightly awkward to type, and despite my previous praise of the +letter `x`, I kind of thought it was pretty lame. Being someone who really +likes Rust, I wanted to call it "rustgrep" or maybe "rgrep" for short. But I +thought that was just as lame, and maybe a little too in-your-face. But I +wanted to continue using `r` so I could at least pretend Rust had something to +do with it. + +I spent a couple of days trying to think of very short words that began with +the letter `r` that were even somewhat related to the task of searching. I +don't remember how it popped into my head, but "rip" came up as something that +meant "fast," as in, "to rip through your text." The fact that RIP is also +an initialism for "Rest in Peace" (as in, "ripgrep kills grep") never really +dawned on me. Perhaps the coincidence is too striking to believe that, but +I didn't realize it until someone explicitly pointed it out to me after the +initial public release. I admit that I found it mildly amusing, but if I had +realized it myself before the public release, I probably would have pressed on +and chose a different name. Alas, renaming things after a release is hard, so I +decided to mush on. + +Given the fact that +[ripgrep never was, is or will be a 100% drop-in replacement for +grep](#posix4ever), +ripgrep is neither actually a "grep killer" nor was it ever intended to be. It +certainly does eat into some of its use cases, but that's nothing that other +tools like ack or The Silver Searcher weren't already doing. + + +

+How can I donate to ripgrep or its maintainers? +

+ +I welcome [sponsorship](https://github.com/sponsors/BurntSushi/). + +Or if you'd prefer, donating to a charitable organization that you like would +also be most welcome. My favorites are: + +* [The Internet Archive](https://archive.org/donate/) +* [Rails Girls](https://railsgirlssummerofcode.org/) +* [Wikipedia](https://wikimediafoundation.org/support/) diff --git a/userland/ripgrep/GUIDE.md b/userland/ripgrep/GUIDE.md new file mode 100644 index 0000000..ca95112 --- /dev/null +++ b/userland/ripgrep/GUIDE.md @@ -0,0 +1,1025 @@ +## User Guide + +This guide is intended to give an elementary description of ripgrep and an +overview of its capabilities. This guide assumes that ripgrep is +[installed](README.md#installation) +and that readers have passing familiarity with using command line tools. This +also assumes a Unix-like system, although most commands are probably easily +translatable to any command line shell environment. + + +### Table of Contents + +* [Basics](#basics) +* [Recursive search](#recursive-search) +* [Automatic filtering](#automatic-filtering) +* [Manual filtering: globs](#manual-filtering-globs) +* [Manual filtering: file types](#manual-filtering-file-types) +* [Replacements](#replacements) +* [Configuration file](#configuration-file) +* [File encoding](#file-encoding) +* [Binary data](#binary-data) +* [Preprocessor](#preprocessor) +* [Common options](#common-options) + + +### Basics + +ripgrep is a command line tool that searches your files for patterns that +you give it. ripgrep behaves as if reading each file line by line. If a line +matches the pattern provided to ripgrep, then that line will be printed. If a +line does not match the pattern, then the line is not printed. + +The best way to see how this works is with an example. To show an example, we +need something to search. Let's try searching ripgrep's source code. First +grab a ripgrep source archive from +https://github.com/BurntSushi/ripgrep/archive/0.7.1.zip +and extract it: + +``` +$ curl -LO https://github.com/BurntSushi/ripgrep/archive/0.7.1.zip +$ unzip 0.7.1.zip +$ cd ripgrep-0.7.1 +$ ls +benchsuite grep tests Cargo.toml LICENSE-MIT +ci ignore wincolor CHANGELOG.md README.md +complete pkg appveyor.yml compile snapcraft.yaml +doc src build.rs COPYING UNLICENSE +globset termcolor Cargo.lock HomebrewFormula +``` + +Let's try our first search by looking for all occurrences of the word `fast` +in `README.md`: + +``` +$ rg fast README.md +75: faster than both. (N.B. It is not, strictly speaking, a "drop-in" replacement +88: color and full Unicode support. Unlike GNU grep, `ripgrep` stays fast while +119:### Is it really faster than everything else? +124:Summarizing, `ripgrep` is fast because: +129: optimizations to make searching very fast. +``` + +(**Note:** If you see an error message from ripgrep saying that it didn't +search any files, then re-run ripgrep with the `--debug` flag. One likely cause +of this is that you have a `*` rule in a `$HOME/.gitignore` file.) + +So what happened here? ripgrep read the contents of `README.md`, and for each +line that contained `fast`, ripgrep printed it to your terminal. ripgrep also +included the line number for each line by default. If your terminal supports +colors, then your output might actually look something like this screenshot: + +[![A screenshot of a sample search ripgrep](https://burntsushi.net/stuff/ripgrep-guide-sample.png)](https://burntsushi.net/stuff/ripgrep-guide-sample.png) + +In this example, we searched for something called a "literal" string. This +means that our pattern was just some normal text that we asked ripgrep to +find. But ripgrep supports the ability to specify patterns via [regular +expressions](https://en.wikipedia.org/wiki/Regular_expression). As an example, +what if we wanted to find all lines have a word that contains `fast` followed +by some number of other letters? + +``` +$ rg 'fast\w+' README.md +75: faster than both. (N.B. It is not, strictly speaking, a "drop-in" replacement +119:### Is it really faster than everything else? +``` + +In this example, we used the pattern `fast\w+`. This pattern tells ripgrep to +look for any lines containing the letters `fast` followed by *one or more* +word-like characters. Namely, `\w` matches characters that compose words (like +`a` and `L` but unlike `.` and ` `). The `+` after the `\w` means, "match the +previous pattern one or more times." This means that the word `fast` won't +match because there are no word characters following the final `t`. But a word +like `faster` will. `faste` would also match! + +Here's a different variation on this same theme: + +``` +$ rg 'fast\w*' README.md +75: faster than both. (N.B. It is not, strictly speaking, a "drop-in" replacement +88: color and full Unicode support. Unlike GNU grep, `ripgrep` stays fast while +119:### Is it really faster than everything else? +124:Summarizing, `ripgrep` is fast because: +129: optimizations to make searching very fast. +``` + +In this case, we used `fast\w*` for our pattern instead of `fast\w+`. The `*` +means that it should match *zero* or more times. In this case, ripgrep will +print the same lines as the pattern `fast`, but if your terminal supports +colors, you'll notice that `faster` will be highlighted instead of just the +`fast` prefix. + +It is beyond the scope of this guide to provide a full tutorial on regular +expressions, but ripgrep's specific syntax is documented here: +https://docs.rs/regex/*/regex/#syntax + + +### Recursive search + +In the previous section, we showed how to use ripgrep to search a single file. +In this section, we'll show how to use ripgrep to search an entire directory +of files. In fact, *recursively* searching your current working directory is +the default mode of operation for ripgrep, which means doing this is very +simple. + +Using our unzipped archive of ripgrep source code, here's how to find all +function definitions whose name is `write`: + +``` +$ rg 'fn write\(' +src/printer.rs +469: fn write(&mut self, buf: &[u8]) { + +termcolor/src/lib.rs +227: fn write(&mut self, b: &[u8]) -> io::Result { +250: fn write(&mut self, b: &[u8]) -> io::Result { +428: fn write(&mut self, b: &[u8]) -> io::Result { self.wtr.write(b) } +441: fn write(&mut self, b: &[u8]) -> io::Result { self.wtr.write(b) } +454: fn write(&mut self, buf: &[u8]) -> io::Result { +511: fn write(&mut self, buf: &[u8]) -> io::Result { +848: fn write(&mut self, buf: &[u8]) -> io::Result { +915: fn write(&mut self, buf: &[u8]) -> io::Result { +949: fn write(&mut self, buf: &[u8]) -> io::Result { +1114: fn write(&mut self, buf: &[u8]) -> io::Result { +1348: fn write(&mut self, buf: &[u8]) -> io::Result { +1353: fn write(&mut self, buf: &[u8]) -> io::Result { +``` + +(**Note:** We escape the `(` here because `(` has special significance inside +regular expressions. You could also use `rg -F 'fn write('` to achieve the +same thing, where `-F` interprets your pattern as a literal string instead of +a regular expression.) + +In this example, we didn't specify a file at all. Instead, ripgrep defaulted +to searching your current directory in the absence of a path. In general, +`rg foo` is equivalent to `rg foo ./`. + +This particular search showed us results in both the `src` and `termcolor` +directories. The `src` directory is the core ripgrep code where as `termcolor` +is a dependency of ripgrep (and is used by other tools). What if we only wanted +to search core ripgrep code? Well, that's easy, just specify the directory you +want: + +``` +$ rg 'fn write\(' src +src/printer.rs +469: fn write(&mut self, buf: &[u8]) { +``` + +Here, ripgrep limited its search to the `src` directory. Another way of doing +this search would be to `cd` into the `src` directory and simply use `rg 'fn +write\('` again. + + +### Automatic filtering + +After recursive search, ripgrep's most important feature is what it *doesn't* +search. By default, when you search a directory, ripgrep will ignore all of +the following: + +1. Files and directories that match glob patterns in these three categories: + 1. `.gitignore` globs (including global and repo-specific globs). This + includes `.gitignore` files in parent directories that are part of the + same `git` repository. (Unless the `--no-require-git` flag is given.) + 2. `.ignore` globs, which take precedence over all gitignore globs + when there's a conflict. This includes `.ignore` files in parent + directories. + 3. `.rgignore` globs, which take precedence over all `.ignore` globs + when there's a conflict. This includes `.rgignore` files in parent + directories. +2. Hidden files and directories. +3. Binary files. (ripgrep considers any file with a `NUL` byte to be binary.) +4. Symbolic links aren't followed. + +All of these things can be toggled using various flags provided by ripgrep: + +1. You can disable all ignore-related filtering with the `--no-ignore` flag. +2. Hidden files and directories can be searched with the `--hidden` (`-.` for +short) flag. +3. Binary files can be searched via the `--text` (`-a` for short) flag. + Be careful with this flag! Binary files may emit control characters to your + terminal, which might cause strange behavior. +4. ripgrep can follow symlinks with the `--follow` (`-L` for short) flag. + +As a special convenience, ripgrep also provides a flag called `--unrestricted` +(`-u` for short). Repeated uses of this flag will cause ripgrep to disable +more and more of its filtering. That is, `-u` will disable `.gitignore` +handling, `-uu` will search hidden files and directories and `-uuu` will search +binary files. This is useful when you're using ripgrep and you aren't sure +whether its filtering is hiding results from you. Tacking on a couple `-u` +flags is a quick way to find out. (Use the `--debug` flag if you're still +perplexed, and if that doesn't help, +[file an issue](https://github.com/BurntSushi/ripgrep/issues/new).) + +ripgrep's `.gitignore` handling actually goes a bit beyond just `.gitignore` +files. ripgrep will also respect repository specific rules found in +`$GIT_DIR/info/exclude`, as well as any global ignore rules in your +`core.excludesFile` (which is usually `$XDG_CONFIG_HOME/git/ignore` on +Unix-like systems). + +Sometimes you want to search files that are in your `.gitignore`, so it is +possible to specify additional ignore rules or overrides in a `.ignore` +(application agnostic) or `.rgignore` (ripgrep specific) file. + +For example, let's say you have a `.gitignore` file that looks like this: + +``` +log/ +``` + +This generally means that any `log` directory won't be tracked by `git`. +However, perhaps it contains useful output that you'd like to include in your +searches, but you still don't want to track it in `git`. You can achieve this +by creating a `.ignore` file in the same directory as the `.gitignore` file +with the following contents: + +``` +!log/ +``` + +ripgrep treats `.ignore` files with higher precedence than `.gitignore` files +(and treats `.rgignore` files with higher precedence than `.ignore` files). +This means ripgrep will see the `!log/` whitelist rule first and search that +directory. + +Like `.gitignore`, a `.ignore` file can be placed in any directory. Its rules +will be processed with respect to the directory it resides in, just like +`.gitignore`. + +To process `.gitignore` and `.ignore` files case insensitively, use the flag +`--ignore-file-case-insensitive`. This is especially useful on case insensitive +file systems like those on Windows and macOS. Note though that this can come +with a significant performance penalty, and is therefore disabled by default. + +For a more in depth description of how glob patterns in a `.gitignore` file +are interpreted, please see `man gitignore`. + + +### Manual filtering: globs + +In the previous section, we talked about ripgrep's filtering that it does by +default. It is "automatic" because it reacts to your environment. That is, it +uses already existing `.gitignore` files to produce more relevant search +results. + +In addition to automatic filtering, ripgrep also provides more manual or ad hoc +filtering. This comes in two varieties: additional glob patterns specified in +your ripgrep commands and file type filtering. This section covers glob +patterns while the next section covers file type filtering. + +In our ripgrep source code (see [Basics](#basics) for instructions on how to +get a source archive to search), let's say we wanted to see which things depend +on `lexopt`, our argument parser. + +We could do this: + +``` +$ rg lexopt +[lots of results] +``` + +But this shows us many things, and we're only interested in where we wrote +`lexopt` as a dependency. Instead, we could limit ourselves to TOML files, +which is how dependencies are communicated to Rust's build tool, Cargo: + +``` +$ rg lexopt -g '*.toml' +Cargo.toml +57:lexopt = "0.3.0" +``` + +The `-g '*.toml'` syntax says, "make sure every file searched matches this +glob pattern." Note that we put `'*.toml'` in single quotes to prevent our +shell from expanding the `*`. + +If we wanted, we could tell ripgrep to search anything *but* `*.toml` files: + +``` +$ rg lexopt -g '!*.toml' +[lots of results] +``` + +This will give you a lot of results again as above, but they won't include +files ending with `.toml`. Note that the use of a `!` here to mean "negation" +is a bit non-standard, but it was chosen to be consistent with how globs in +`.gitignore` files are written. (Although, the meaning is reversed. In +`.gitignore` files, a `!` prefix means whitelist, and on the command line, a +`!` means blacklist.) + +Globs are interpreted in exactly the same way as `.gitignore` patterns. That +is, later globs will override earlier globs. For example, the following command +will search only `*.toml` files: + +``` +$ rg lexopt -g '!*.toml' -g '*.toml' +``` + +Interestingly, reversing the order of the globs in this case will match +nothing, since the presence of at least one non-blacklist glob will institute a +requirement that every file searched must match at least one glob. In this +case, the blacklist glob takes precedence over the previous glob and prevents +any file from being searched at all! + + +### Manual filtering: file types + +Over time, you might notice that you use the same glob patterns over and over. +For example, you might find yourself doing a lot of searches where you only +want to see results for Rust files: + +``` +$ rg 'fn run' -g '*.rs' +``` + +Instead of writing out the glob every time, you can use ripgrep's support for +file types: + +``` +$ rg 'fn run' --type rust +``` + +or, more succinctly, + +``` +$ rg 'fn run' -trust +``` + +The way the `--type` flag functions is simple. It acts as a name that is +assigned to one or more globs that match the relevant files. This lets you +write a single type that might encompass a broad range of file extensions. For +example, if you wanted to search C files, you'd have to check both C source +files and C header files: + +``` +$ rg 'int main' -g '*.{c,h}' +``` + +or you could just use the C file type: + +``` +$ rg 'int main' -tc +``` + +Just as you can write blacklist globs, you can blacklist file types too: + +``` +$ rg lexopt --type-not rust +``` + +or, more succinctly, + +``` +$ rg lexopt -Trust +``` + +That is, `-t` means "include files of this type" where as `-T` means "exclude +files of this type." + +To see the globs that make up a type, run `rg --type-list`: + +``` +$ rg --type-list | rg '^make:' +make: *.mak, *.mk, GNUmakefile, Gnumakefile, Makefile, gnumakefile, makefile +``` + +By default, ripgrep comes with a bunch of pre-defined types. Generally, these +types correspond to well known public formats. But you can define your own +types as well. For example, perhaps you frequently search "web" files, which +consist of JavaScript, HTML and CSS: + +``` +$ rg --type-add 'web:*.html' --type-add 'web:*.css' --type-add 'web:*.js' -tweb title +``` + +or, more succinctly, + +``` +$ rg --type-add 'web:*.{html,css,js}' -tweb title +``` + +The above command defines a new type, `web`, corresponding to the glob +`*.{html,css,js}`. It then applies the new filter with `-tweb` and searches for +the pattern `title`. If you ran + +``` +$ rg --type-add 'web:*.{html,css,js}' --type-list +``` + +Then you would see your `web` type show up in the list, even though it is not +part of ripgrep's built-in types. + +It is important to stress here that the `--type-add` flag only applies to the +current command. It does not add a new file type and save it somewhere in a +persistent form. If you want a type to be available in every ripgrep command, +then you should either create a shell alias: + +``` +alias rg="rg --type-add 'web:*.{html,css,js}'" +``` + +or add `--type-add=web:*.{html,css,js}` to your ripgrep configuration file. +([Configuration files](#configuration-file) are covered in more detail later.) + +#### The special `all` file type + +A special option supported by the `--type` flag is `all`. `--type all` looks +for a match in any of the supported file types listed by `--type-list`, +including those added on the command line using `--type-add`. It's equivalent +to the command `rg --type agda --type asciidoc --type asm ...`, where `...` +stands for a list of `--type` flags for the rest of the types in `--type-list`. + +As an example, let's suppose you have a shell script in your current directory, +`my-shell-script`, which includes a shell library, `my-shell-library.bash`. +Both `rg --type sh` and `rg --type all` would only search for matches in +`my-shell-library.bash`, not `my-shell-script`, because the globs matched +by the `sh` file type don't include files without an extension. On the +other hand, `rg --type-not all` would search `my-shell-script` but not +`my-shell-library.bash`. + +### Replacements + +ripgrep provides a limited ability to modify its output by replacing matched +text with some other text. This is easiest to explain with an example. Remember +when we searched for the word `fast` in ripgrep's README? + +``` +$ rg fast README.md +75: faster than both. (N.B. It is not, strictly speaking, a "drop-in" replacement +88: color and full Unicode support. Unlike GNU grep, `ripgrep` stays fast while +119:### Is it really faster than everything else? +124:Summarizing, `ripgrep` is fast because: +129: optimizations to make searching very fast. +``` + +What if we wanted to *replace* all occurrences of `fast` with `FAST`? That's +easy with ripgrep's `--replace` flag: + +``` +$ rg fast README.md --replace FAST +75: FASTer than both. (N.B. It is not, strictly speaking, a "drop-in" replacement +88: color and full Unicode support. Unlike GNU grep, `ripgrep` stays FAST while +119:### Is it really FASTer than everything else? +124:Summarizing, `ripgrep` is FAST because: +129: optimizations to make searching very FAST. +``` + +or, more succinctly, + +``` +$ rg fast README.md -r FAST +[snip] +``` + +In essence, the `--replace` flag applies *only* to the matching portion of text +in the output. If you instead wanted to replace an entire line of text, then +you need to include the entire line in your match. For example: + +``` +$ rg '^.*fast.*$' README.md -r FAST +75:FAST +88:FAST +119:FAST +124:FAST +129:FAST +``` + +Alternatively, you can combine the `--only-matching` (or `-o` for short) with +the `--replace` flag to achieve the same result: + +``` +$ rg fast README.md --only-matching --replace FAST +75:FAST +88:FAST +119:FAST +124:FAST +129:FAST +``` + +or, more succinctly, + +``` +$ rg fast README.md -or FAST +[snip] +``` + +Finally, replacements can include capturing groups. For example, let's say +we wanted to find all occurrences of `fast` followed by another word and +join them together with a dash. The pattern we might use for that is +`fast\s+(\w+)`, which matches `fast`, followed by any amount of whitespace, +followed by any number of "word" characters. We put the `\w+` in a "capturing +group" (indicated by parentheses) so that we can reference it later in our +replacement string. For example: + +``` +$ rg 'fast\s+(\w+)' README.md -r 'fast-$1' +88: color and full Unicode support. Unlike GNU grep, `ripgrep` stays fast-while +124:Summarizing, `ripgrep` is fast-because: +``` + +Our replacement string here, `fast-$1`, consists of `fast-` followed by the +contents of the capturing group at index `1`. (Capturing groups actually start +at index 0, but the `0`th capturing group always corresponds to the entire +match. The capturing group at index `1` always corresponds to the first +explicit capturing group found in the regex pattern.) + +Capturing groups can also be named, which is sometimes more convenient than +using the indices. For example, the following command is equivalent to the +above command: + +``` +$ rg 'fast\s+(?P\w+)' README.md -r 'fast-$word' +88: color and full Unicode support. Unlike GNU grep, `ripgrep` stays fast-while +124:Summarizing, `ripgrep` is fast-because: +``` + +It is important to note that ripgrep **will never modify your files**. The +`--replace` flag only controls ripgrep's output. (And there is no flag to let +you do a replacement in a file.) + + +### Configuration file + +It is possible that ripgrep's default options aren't suitable in every case. +For that reason, and because shell aliases aren't always convenient, ripgrep +supports configuration files. + +Setting up a configuration file is simple. ripgrep will not look in any +predetermined directory for a config file automatically. Instead, you need to +set the `RIPGREP_CONFIG_PATH` environment variable to the file path of your +config file. Once the environment variable is set, open the file and just type +in the flags you want set automatically. There are only two rules for +describing the format of the config file: + +1. Every line is a shell argument, after trimming whitespace. +2. Lines starting with `#` (optionally preceded by any amount of whitespace) +are ignored. + +In particular, there is no escaping. Each line is given to ripgrep as a single +command line argument verbatim. + +Here's an example of a configuration file, which demonstrates some of the +formatting peculiarities: + +``` +$ cat $HOME/.ripgreprc +# Don't let ripgrep vomit really long lines to my terminal, and show a preview. +--max-columns=150 +--max-columns-preview + +# Add my 'web' type. +--type-add +web:*.{html,css,js}* + +# Search hidden files / directories (e.g. dotfiles) by default +--hidden + +# Using glob patterns to include/exclude files or folders +--glob=!.git/* + +# or +--glob +!.git/* + +# Set the colors. +--colors=line:none +--colors=line:style:bold + +# Because who cares about case!? +--smart-case +``` + +When we use a flag that has a value, we either put the flag and the value on +the same line but delimited by an `=` sign (e.g., `--max-columns=150`), or we +put the flag and the value on two different lines. This is because ripgrep's +argument parser knows to treat the single argument `--max-columns=150` as a +flag with a value, but if we had written `--max-columns 150` in our +configuration file, then ripgrep's argument parser wouldn't know what to do +with it. + +Putting the flag and value on different lines is exactly equivalent and is a +matter of style. + +Comments are encouraged so that you remember what the config is doing. Empty +lines are OK too. + +So let's say you're using the above configuration file, but while you're at a +terminal, you really want to be able to see lines longer than 150 columns. What +do you do? Thankfully, all you need to do is pass `--max-columns 0` (or `-M0` +for short) on the command line, which will override your configuration file's +setting. This works because ripgrep's configuration file is *prepended* to the +explicit arguments you give it on the command line. Since flags given later +override flags given earlier, everything works as expected. This works for most +other flags as well, and each flag's documentation states which other flags +override it. + +If you're confused about what configuration file ripgrep is reading arguments +from, then running ripgrep with the `--debug` flag should help clarify things. +The debug output should note what config file is being loaded and the arguments +that have been read from the configuration. + +Finally, if you want to make absolutely sure that ripgrep *isn't* reading a +configuration file, then you can pass the `--no-config` flag, which will always +prevent ripgrep from reading extraneous configuration from the environment, +regardless of what other methods of configuration are added to ripgrep in the +future. + + +### File encoding + +[Text encoding](https://en.wikipedia.org/wiki/Character_encoding) is a complex +topic, but we can try to summarize its relevancy to ripgrep: + +* Files are generally just a bundle of bytes. There is no reliable way to know + their encoding. +* Either the encoding of the pattern must match the encoding of the files being + searched, or a form of transcoding must be performed that converts either the + pattern or the file to the same encoding as the other. +* ripgrep tends to work best on plain text files, and among plain text files, + the most popular encodings likely consist of ASCII, latin1 or UTF-8. As + a special exception, UTF-16 is prevalent in Windows environments + +In light of the above, here is how ripgrep behaves when `--encoding auto` is +given, which is the default: + +* All input is assumed to be ASCII compatible (which means every byte that + corresponds to an ASCII codepoint actually is an ASCII codepoint). This + includes ASCII itself, latin1 and UTF-8. +* ripgrep works best with UTF-8. For example, ripgrep's regular expression + engine supports Unicode features. Namely, character classes like `\w` will + match all word characters by Unicode's definition and `.` will match any + Unicode codepoint instead of any byte. These constructions assume UTF-8, + so they simply won't match when they come across bytes in a file that aren't + UTF-8. +* To handle the UTF-16 case, ripgrep will do something called "BOM sniffing" + by default. That is, the first three bytes of a file will be read, and if + they correspond to a UTF-16 BOM, then ripgrep will transcode the contents of + the file from UTF-16 to UTF-8, and then execute the search on the transcoded + version of the file. (This incurs a performance penalty since transcoding + is needed in addition to regex searching.) If the file contains invalid + UTF-16, then the Unicode replacement codepoint is substituted in place of + invalid code units. +* To handle other cases, ripgrep provides a `-E/--encoding` flag, which permits + you to specify an encoding from the + [Encoding Standard](https://encoding.spec.whatwg.org/#concept-encoding-get). + ripgrep will assume *all* files searched are the encoding specified (unless + the file has a BOM) and will perform a transcoding step just like in the + UTF-16 case described above. + +By default, ripgrep will not require its input be valid UTF-8. That is, ripgrep +can and will search arbitrary bytes. The key here is that if you're searching +content that isn't UTF-8, then the usefulness of your pattern will degrade. If +you're searching bytes that aren't ASCII compatible, then it's likely the +pattern won't find anything. With all that said, this mode of operation is +important, because it lets you find ASCII or UTF-8 *within* files that are +otherwise arbitrary bytes. + +As a special case, the `-E/--encoding` flag supports the value `none`, which +will completely disable all encoding related logic, including BOM sniffing. +When `-E/--encoding` is set to `none`, ripgrep will search the raw bytes of +the underlying file with no transcoding step. For example, here's how you might +search the raw UTF-16 encoding of the string `Шерлок`: + +``` +$ rg '(?-u)\(\x045\x04@\x04;\x04>\x04:\x04' -E none -a some-utf16-file +``` + +Of course, that's just an example meant to show how one can drop down into +raw bytes. Namely, the simpler command works as you might expect automatically: + +``` +$ rg 'Шерлок' some-utf16-file +``` + +Finally, it is possible to disable ripgrep's Unicode support from within the +regular expression. For example, let's say you wanted `.` to match any byte +rather than any Unicode codepoint. (You might want this while searching a +binary file, since `.` by default will not match invalid UTF-8.) You could do +this by disabling Unicode via a regular expression flag: + +``` +$ rg '(?-u:.)' +``` + +This works for any part of the pattern. For example, the following will find +any Unicode word character followed by any ASCII word character followed by +another Unicode word character: + +``` +$ rg '\w(?-u:\w)\w' +``` + + +### Binary data + +In addition to skipping hidden files and files in your `.gitignore` by default, +ripgrep also attempts to skip binary files. ripgrep does this by default +because binary files (like PDFs or images) are typically not things you want to +search when searching for regex matches. Moreover, if content in a binary file +did match, then it's possible for undesirable binary data to be printed to your +terminal and wreak havoc. + +Unfortunately, unlike skipping hidden files and respecting your `.gitignore` +rules, a file cannot as easily be classified as binary. In order to figure out +whether a file is binary, the most effective heuristic that balances +correctness with performance is to simply look for `NUL` bytes. At that point, +the determination is simple: a file is considered "binary" if and only if it +contains a `NUL` byte somewhere in its contents. + +The issue is that while most binary files will have a `NUL` byte toward the +beginning of its contents, this is not necessarily true. The `NUL` byte might +be the very last byte in a large file, but that file is still considered +binary. While this leads to a fair amount of complexity inside ripgrep's +implementation, it also results in some unintuitive user experiences. + +At a high level, ripgrep operates in three different modes with respect to +binary files: + +1. The default mode is to attempt to remove binary files from a search + completely. This is meant to mirror how ripgrep removes hidden files and + files in your `.gitignore` automatically. That is, as soon as a file is + detected as binary, searching stops. If a match was already printed (because + it was detected long before a `NUL` byte), then ripgrep will print a warning + message indicating that the search stopped prematurely. This default mode + **only applies to files searched by ripgrep as a result of recursive + directory traversal**, which is consistent with ripgrep's other automatic + filtering. For example, `rg foo .file` will search `.file` even though it + is hidden. Similarly, `rg foo binary-file` will search `binary-file` in + "binary" mode automatically. +2. Binary mode is similar to the default mode, except it will not always + stop searching after it sees a `NUL` byte. Namely, in this mode, ripgrep + will continue searching a file that is known to be binary until the first + of two conditions is met: 1) the end of the file has been reached or 2) a + match is or has been seen. This means that in binary mode, if ripgrep + reports no matches, then there are no matches in the file. When a match does + occur, ripgrep prints a message similar to one it prints when in its default + mode indicating that the search has stopped prematurely. This mode can be + forcefully enabled for all files with the `--binary` flag. The purpose of + binary mode is to provide a way to discover matches in all files, but to + avoid having binary data dumped into your terminal. +3. Text mode completely disables all binary detection and searches all files + as if they were text. This is useful when searching a file that is + predominantly text but contains a `NUL` byte, or if you are specifically + trying to search binary data. This mode can be enabled with the `-a/--text` + flag. Note that when using this mode on very large binary files, it is + possible for ripgrep to use a lot of memory. + +Unfortunately, there is one additional complexity in ripgrep that can make it +difficult to reason about binary files. That is, the way binary detection works +depends on the way that ripgrep searches your files. Specifically: + +* When ripgrep uses memory maps, then binary detection is only performed on the + first few kilobytes of the file in addition to every matching line. +* When ripgrep doesn't use memory maps, then binary detection is performed on + all bytes searched. + +This means that whether a file is detected as binary or not can change based +on the internal search strategy used by ripgrep. If you prefer to keep +ripgrep's binary file detection consistent, then you can disable memory maps +via the `--no-mmap` flag. (The cost will be a small performance regression when +searching very large files on some platforms.) + + +### Preprocessor + +In ripgrep, a preprocessor is any type of command that can be run to transform +the input of every file before ripgrep searches it. This makes it possible to +search virtually any kind of content that can be automatically converted to +text without having to teach ripgrep how to read said content. + +One common example is searching PDFs. PDFs are first and foremost meant to be +displayed to users. But PDFs often have text streams in them that can be useful +to search. In our case, we want to search Bruce Watson's excellent +dissertation, +[Taxonomies and Toolkits of Regular Language Algorithms](https://burntsushi.net/stuff/1995-watson.pdf). +After downloading it, let's try searching it: + +``` +$ rg 'The Commentz-Walter algorithm' 1995-watson.pdf +$ +``` + +Surely, a dissertation on regular language algorithms would mention +Commentz-Walter. Indeed it does, but our search isn't picking it up because +PDFs are a binary format, and the text shown in the PDF may not be encoded as +simple contiguous UTF-8. Namely, even passing the `-a/--text` flag to ripgrep +will not make our search work. + +One way to fix this is to convert the PDF to plain text first. This won't work +well for all PDFs, but does great in a lot of cases. (Note that the tool we +use, `pdftotext`, is part of the [poppler](https://poppler.freedesktop.org) +PDF rendering library.) + +``` +$ pdftotext 1995-watson.pdf > 1995-watson.txt +$ rg 'The Commentz-Walter algorithm' 1995-watson.txt +316:The Commentz-Walter algorithms : : : : : : : : : : : : : : : +7165:4.4 The Commentz-Walter algorithms +10062:in input string S , we obtain the Boyer-Moore algorithm. The Commentz-Walter algorithm +17218:The Commentz-Walter algorithm (and its variants) displayed more interesting behaviour, +17249:Aho-Corasick algorithms are used extensively. The Commentz-Walter algorithms are used +17297: The Commentz-Walter algorithms (CW). In all versions of the CW algorithms, a common program skeleton is used with di erent shift functions. The CW algorithms are +``` + +But having to explicitly convert every file can be a pain, especially when you +have a directory full of PDF files. Instead, we can use ripgrep's preprocessor +feature to search the PDF. ripgrep's `--pre` flag works by taking a single +command name and then executing that command for every file that it searches. +ripgrep passes the file path as the first and only argument to the command and +also sends the contents of the file to stdin. So let's write a simple shell +script that wraps `pdftotext` in a way that conforms to this interface: + +``` +$ cat preprocess +#!/bin/sh + +exec pdftotext - - +``` + +With `preprocess` in the same directory as `1995-watson.pdf`, we can now use it +to search the PDF: + +``` +$ rg --pre ./preprocess 'The Commentz-Walter algorithm' 1995-watson.pdf +316:The Commentz-Walter algorithms : : : : : : : : : : : : : : : +7165:4.4 The Commentz-Walter algorithms +10062:in input string S , we obtain the Boyer-Moore algorithm. The Commentz-Walter algorithm +17218:The Commentz-Walter algorithm (and its variants) displayed more interesting behaviour, +17249:Aho-Corasick algorithms are used extensively. The Commentz-Walter algorithms are used +17297: The Commentz-Walter algorithms (CW). In all versions of the CW algorithms, a common program skeleton is used with di erent shift functions. The CW algorithms are +``` + +Note that `preprocess` must be resolvable to a command that ripgrep can read. +The simplest way to do this is to put your preprocessor command in a directory +that is in your `PATH` (or equivalent), or otherwise use an absolute path. + +As a bonus, this turns out to be quite a bit faster than other specialized PDF +grepping tools: + +``` +$ time rg --pre ./preprocess 'The Commentz-Walter algorithm' 1995-watson.pdf -c +6 + +real 0.697 +user 0.684 +sys 0.007 +maxmem 16 MB +faults 0 + +$ time pdfgrep 'The Commentz-Walter algorithm' 1995-watson.pdf -c +6 + +real 1.336 +user 1.310 +sys 0.023 +maxmem 16 MB +faults 0 +``` + +If you wind up needing to search a lot of PDFs, then ripgrep's parallelism can +make the speed difference even greater. + +#### A more robust preprocessor + +One of the problems with the aforementioned preprocessor is that it will fail +if you try to search a file that isn't a PDF: + +``` +$ echo foo > not-a-pdf +$ rg --pre ./preprocess 'The Commentz-Walter algorithm' not-a-pdf +not-a-pdf: preprocessor command failed: '"./preprocess" "not-a-pdf"': +------------------------------------------------------------------------------- +Syntax Warning: May not be a PDF file (continuing anyway) +Syntax Error: Couldn't find trailer dictionary +Syntax Error: Couldn't find trailer dictionary +Syntax Error: Couldn't read xref table +``` + +To fix this, we can make our preprocessor script a bit more robust by only +running `pdftotext` when we think the input is a non-empty PDF: + +``` +$ cat preprocessor +#!/bin/sh + +case "$1" in +*.pdf) + # The -s flag ensures that the file is non-empty. + if [ -s "$1" ]; then + exec pdftotext - - + else + exec cat + fi + ;; +*) + exec cat + ;; +esac +``` + +We can even extend our preprocessor to search other kinds of files. Sometimes +we don't always know the file type from the file name, so we can use the `file` +utility to "sniff" the type of the file based on its contents: + +``` +$ cat processor +#!/bin/sh + +case "$1" in +*.pdf) + # The -s flag ensures that the file is non-empty. + if [ -s "$1" ]; then + exec pdftotext - - + else + exec cat + fi + ;; +*) + case $(file "$1") in + *Zstandard*) + exec pzstd -cdq + ;; + *) + exec cat + ;; + esac + ;; +esac +``` + +#### Reducing preprocessor overhead + +There is one more problem with the above approach: it requires running a +preprocessor for every single file that ripgrep searches. If every file needs +a preprocessor, then this is OK. But if most don't, then this can substantially +slow down searches because of the overhead of launching new processors. You +can avoid this by telling ripgrep to only invoke the preprocessor when the file +path matches a glob. For example, consider the performance difference even when +searching a repository as small as ripgrep's: + +``` +$ time rg --pre pre-rg 'fn is_empty' -c +crates/globset/src/lib.rs:1 +crates/matcher/src/lib.rs:2 +crates/ignore/src/overrides.rs:1 +crates/ignore/src/gitignore.rs:1 +crates/ignore/src/types.rs:1 + +real 0.138 +user 0.485 +sys 0.209 +maxmem 7 MB +faults 0 + +$ time rg --pre pre-rg --pre-glob '*.pdf' 'fn is_empty' -c +crates/globset/src/lib.rs:1 +crates/ignore/src/types.rs:1 +crates/ignore/src/gitignore.rs:1 +crates/ignore/src/overrides.rs:1 +crates/matcher/src/lib.rs:2 + +real 0.008 +user 0.010 +sys 0.002 +maxmem 7 MB +faults 0 +``` + + +### Common options + +ripgrep has a lot of flags. Too many to keep in your head at once. This section +is intended to give you a sampling of some of the most important and frequently +used options that will likely impact how you use ripgrep on a regular basis. + +* `-h`: Show ripgrep's condensed help output. +* `--help`: Show ripgrep's longer form help output. (Nearly what you'd find in + ripgrep's man page, so pipe it into a pager!) +* `-i/--ignore-case`: When searching for a pattern, ignore case differences. + That is `rg -i fast` matches `fast`, `fASt`, `FAST`, etc. +* `-S/--smart-case`: This is similar to `--ignore-case`, but disables itself + if the pattern contains any uppercase letters. Usually this flag is put into + alias or a config file. +* `-F/--fixed-strings`: Disable regular expression matching and treat the pattern + as a literal string. +* `-w/--word-regexp`: Require that all matches of the pattern be surrounded + by word boundaries. That is, given `pattern`, the `--word-regexp` flag will + cause ripgrep to behave as if `pattern` were actually + `\b{start-half}(?:pattern)\b{end-half}`. + (Unlike `\b`, these half-boundaries don't require a word character on one + side. For example, `rg -w -e -2` will match `-2` in `(-2)` but `rg '\b-2\b'` + will not.) +* `-c/--count`: Report a count of total matched lines. +* `--files`: Print the files that ripgrep *would* search, but don't actually + search them. +* `-a/--text`: Search binary files as if they were plain text. +* `-U/--multiline`: Permit matches to span multiple lines. +* `-z/--search-zip`: Search compressed files (gzip, bzip2, lzma, xz, lz4, + brotli, zstd). This is disabled by default. +* `-C/--context`: Show the lines surrounding a match. +* `--sort path`: Force ripgrep to sort its output by file name. (This disables + parallelism, so it might be slower.) +* `-L/--follow`: Follow symbolic links while recursively searching. +* `-M/--max-columns`: Limit the length of lines printed by ripgrep. +* `--debug`: Shows ripgrep's debug output. This is useful for understanding + why a particular file might be ignored from search, or what kinds of + configuration ripgrep is loading from the environment. diff --git a/userland/ripgrep/LICENSE-MIT b/userland/ripgrep/LICENSE-MIT new file mode 100644 index 0000000..3b0a5dc --- /dev/null +++ b/userland/ripgrep/LICENSE-MIT @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2015 Andrew Gallant + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/userland/ripgrep/PATCHES.md b/userland/ripgrep/PATCHES.md new file mode 100644 index 0000000..f4b8dbd --- /dev/null +++ b/userland/ripgrep/PATCHES.md @@ -0,0 +1,9 @@ +# WASI patch + +Source: crates.io `ripgrep` 15.2.0, https://github.com/BurntSushi/ripgrep. +The vendor copy retains the upstream licenses. + +`crates/core/flags/hiargs.rs` forces one search thread on WASI. +This uses the existing sequential walker and search path. +Explicit `-j` values cannot enable threads on WASI. +All other source files retain their upstream behavior. diff --git a/userland/ripgrep/README.md b/userland/ripgrep/README.md new file mode 100644 index 0000000..73868ac --- /dev/null +++ b/userland/ripgrep/README.md @@ -0,0 +1,541 @@ +ripgrep (rg) +------------ +ripgrep is a line-oriented search tool that recursively searches the current +directory for a regex pattern. By default, ripgrep will respect gitignore rules +and automatically skip hidden files/directories and binary files. (To disable +all automatic filtering by default, use `rg -uuu`.) ripgrep has first class +support on Windows, macOS and Linux, with binary downloads available for [every +release](https://github.com/BurntSushi/ripgrep/releases). ripgrep is similar to +other popular search tools like The Silver Searcher, ack and grep. + +[![Build status](https://github.com/BurntSushi/ripgrep/workflows/ci/badge.svg)](https://github.com/BurntSushi/ripgrep/actions) +[![Crates.io](https://img.shields.io/crates/v/ripgrep.svg)](https://crates.io/crates/ripgrep) +[![Packaging status](https://repology.org/badge/tiny-repos/ripgrep.svg)](https://repology.org/project/ripgrep/badges) + +Dual-licensed under MIT or the [UNLICENSE](https://unlicense.org). + + +### CHANGELOG + +Please see the [CHANGELOG](CHANGELOG.md) for a release history. + +### Documentation quick links + +* [Installation](#installation) +* [User Guide](GUIDE.md) +* [Frequently Asked Questions](FAQ.md) +* [Regex syntax](https://docs.rs/regex/1/regex/#syntax) +* [Configuration files](GUIDE.md#configuration-file) +* [Shell completions](FAQ.md#complete) +* [Building](#building) +* [Translations](#translations) + + +### Screenshot of search results + +[![A screenshot of a sample search with ripgrep](https://burntsushi.net/stuff/ripgrep1.png)](https://burntsushi.net/stuff/ripgrep1.png) + + +### Quick examples comparing tools + +This example searches the entire +[Linux kernel source tree](https://github.com/BurntSushi/linux) +(after running `make defconfig && make -j8`) for `[A-Z]+_SUSPEND`, where +all matches must be words. Timings were collected on a system with an Intel +i9-12900K 5.2 GHz. + +Please remember that a single benchmark is never enough! See my +[blog post on ripgrep](https://blog.burntsushi.net/ripgrep/) +for a very detailed comparison with more benchmarks and analysis. + +| Tool | Command | Line count | Time | +| ---- | ------- | ---------- | ---- | +| ripgrep (Unicode) | `rg -n -w '[A-Z]+_SUSPEND'` | 536 | **0.082s** (1.00x) | +| [hypergrep](https://github.com/p-ranav/hypergrep) | `hgrep -n -w '[A-Z]+_SUSPEND'` | 536 | 0.167s (2.04x) | +| [git grep](https://www.kernel.org/pub/software/scm/git/docs/git-grep.html) | `git grep -P -n -w '[A-Z]+_SUSPEND'` | 536 | 0.273s (3.34x) | +| [The Silver Searcher](https://github.com/ggreer/the_silver_searcher) | `ag -w '[A-Z]+_SUSPEND'` | 534 | 0.443s (5.43x) | +| [ugrep](https://github.com/Genivia/ugrep) | `ugrep -r --ignore-files --no-hidden -I -w '[A-Z]+_SUSPEND'` | 536 | 0.639s (7.82x) | +| [git grep](https://www.kernel.org/pub/software/scm/git/docs/git-grep.html) | `LC_ALL=C git grep -E -n -w '[A-Z]+_SUSPEND'` | 536 | 0.727s (8.91x) | +| [git grep (Unicode)](https://www.kernel.org/pub/software/scm/git/docs/git-grep.html) | `LC_ALL=en_US.UTF-8 git grep -E -n -w '[A-Z]+_SUSPEND'` | 536 | 2.670s (32.70x) | +| [ack](https://github.com/beyondgrep/ack3) | `ack -w '[A-Z]+_SUSPEND'` | 2677 | 2.935s (35.94x) | + +Here's another benchmark on the same corpus as above that disregards gitignore +files and searches with a whitelist instead. The corpus is the same as in the +previous benchmark, and the flags passed to each command ensure that they are +doing equivalent work: + +| Tool | Command | Line count | Time | +| ---- | ------- | ---------- | ---- | +| ripgrep | `rg -uuu -tc -n -w '[A-Z]+_SUSPEND'` | 447 | **0.063s** (1.00x) | +| [ugrep](https://github.com/Genivia/ugrep) | `ugrep -r -n --include='*.c' --include='*.h' -w '[A-Z]+_SUSPEND'` | 447 | 0.607s (9.62x) | +| [GNU grep](https://www.gnu.org/software/grep/) | `grep -E -r -n --include='*.c' --include='*.h' -w '[A-Z]+_SUSPEND'` | 447 | 0.674s (10.69x) | + +Now we'll move to searching on single large file. Here is a straight-up +comparison between ripgrep, ugrep and GNU grep on a file cached in memory +(~13GB, [`OpenSubtitles.raw.en.gz`](http://opus.nlpl.eu/download.php?f=OpenSubtitles/v2018/mono/OpenSubtitles.raw.en.gz), decompressed): + +| Tool | Command | Line count | Time | +| ---- | ------- | ---------- | ---- | +| ripgrep (Unicode) | `rg -w 'Sherlock [A-Z]\w+'` | 7882 | **1.042s** (1.00x) | +| [ugrep](https://github.com/Genivia/ugrep) | `ugrep -w 'Sherlock [A-Z]\w+'` | 7882 | 1.339s (1.28x) | +| [GNU grep (Unicode)](https://www.gnu.org/software/grep/) | `LC_ALL=en_US.UTF-8 egrep -w 'Sherlock [A-Z]\w+'` | 7882 | 6.577s (6.31x) | + +In the above benchmark, passing the `-n` flag (for showing line numbers) +increases the times to `1.664s` for ripgrep and `9.484s` for GNU grep. ugrep +times are unaffected by the presence or absence of `-n`. + +Beware of performance cliffs though: + +| Tool | Command | Line count | Time | +| ---- | ------- | ---------- | ---- | +| ripgrep (Unicode) | `rg -w '[A-Z]\w+ Sherlock [A-Z]\w+'` | 485 | **1.053s** (1.00x) | +| [GNU grep (Unicode)](https://www.gnu.org/software/grep/) | `LC_ALL=en_US.UTF-8 grep -E -w '[A-Z]\w+ Sherlock [A-Z]\w+'` | 485 | 6.234s (5.92x) | +| [ugrep](https://github.com/Genivia/ugrep) | `ugrep -w '[A-Z]\w+ Sherlock [A-Z]\w+'` | 485 | 28.973s (27.51x) | + +And performance can drop precipitously across the board when searching big +files for patterns without any opportunities for literal optimizations: + +| Tool | Command | Line count | Time | +| ---- | ------- | ---------- | ---- | +| ripgrep | `rg '[A-Za-z]{30}'` | 6749 | **15.569s** (1.00x) | +| [ugrep](https://github.com/Genivia/ugrep) | `ugrep -E '[A-Za-z]{30}'` | 6749 | 21.857s (1.40x) | +| [GNU grep](https://www.gnu.org/software/grep/) | `LC_ALL=C grep -E '[A-Za-z]{30}'` | 6749 | 32.409s (2.08x) | +| [GNU grep (Unicode)](https://www.gnu.org/software/grep/) | `LC_ALL=en_US.UTF-8 grep -E '[A-Za-z]{30}'` | 6795 | 8m30s (32.74x) | + +Finally, high match counts also tend to both tank performance and smooth +out the differences between tools (because performance is dominated by how +quickly one can handle a match and not the algorithm used to detect the match, +generally speaking): + +| Tool | Command | Line count | Time | +| ---- | ------- | ---------- | ---- | +| ripgrep | `rg the` | 83499915 | **6.948s** (1.00x) | +| [ugrep](https://github.com/Genivia/ugrep) | `ugrep the` | 83499915 | 11.721s (1.69x) | +| [GNU grep](https://www.gnu.org/software/grep/) | `LC_ALL=C grep the` | 83499915 | 15.217s (2.19x) | + +### Why should I use ripgrep? + +* It can replace many use cases served by other search tools + because it contains most of their features and is generally faster. (See + [the FAQ](FAQ.md#posix4ever) for more details on whether ripgrep can truly + replace grep.) +* Like other tools specialized to code search, ripgrep defaults to + [recursive search](GUIDE.md#recursive-search) and does [automatic + filtering](GUIDE.md#automatic-filtering). Namely, ripgrep won't search files + ignored by your `.gitignore`/`.ignore`/`.rgignore` files, it won't search + hidden files and it won't search binary files. Automatic filtering can be + disabled with `rg -uuu`. +* ripgrep can [search specific types of files](GUIDE.md#manual-filtering-file-types). + For example, `rg -tpy foo` limits your search to Python files and `rg -Tjs + foo` excludes JavaScript files from your search. ripgrep can be taught about + new file types with custom matching rules. +* ripgrep supports many features found in `grep`, such as showing the context + of search results, searching multiple patterns, highlighting matches with + color and full Unicode support. Unlike GNU grep, ripgrep stays fast while + supporting Unicode (which is always on). +* ripgrep has optional support for switching its regex engine to use PCRE2. + Among other things, this makes it possible to use look-around and + backreferences in your patterns, which are not supported in ripgrep's default + regex engine. PCRE2 support can be enabled with `-P/--pcre2` (use PCRE2 + always) or `--auto-hybrid-regex` (use PCRE2 only if needed). An alternative + syntax is provided via the `--engine (default|pcre2|auto)` option. +* ripgrep has [rudimentary support for replacements](GUIDE.md#replacements), + which permit rewriting output based on what was matched. +* ripgrep supports [searching files in text encodings](GUIDE.md#file-encoding) + other than UTF-8, such as UTF-16, latin-1, GBK, EUC-JP, Shift_JIS and more. + (Some support for automatically detecting UTF-16 is provided. Other text + encodings must be specifically specified with the `-E/--encoding` flag.) +* ripgrep supports searching files compressed in a common format (brotli, + bzip2, gzip, lz4, lzma, xz, or zstandard) with the `-z/--search-zip` flag. +* ripgrep supports + [arbitrary input preprocessing filters](GUIDE.md#preprocessor) + which could be PDF text extraction, less supported decompression, decrypting, + automatic encoding detection and so on. +* ripgrep can be configured via a + [configuration file](GUIDE.md#configuration-file). + +In other words, use ripgrep if you like speed, filtering by default, fewer +bugs and Unicode support. + + +### Why shouldn't I use ripgrep? + +Despite initially not wanting to add every feature under the sun to ripgrep, +over time, ripgrep has grown support for most features found in other file +searching tools. This includes searching for results spanning across multiple +lines, and opt-in support for PCRE2, which provides look-around and +backreference support. + +At this point, the primary reasons not to use ripgrep probably consist of one +or more of the following: + +* You need a portable and ubiquitous tool. While ripgrep works on Windows, + macOS and Linux, it is not ubiquitous and it does not conform to any + standard such as POSIX. The best tool for this job is good old grep. +* There still exists some other feature (or bug) not listed in this README that + you rely on that's in another tool that isn't in ripgrep. +* There is a performance edge case where ripgrep doesn't do well where another + tool does do well. (Please file a bug report!) +* ripgrep isn't possible to install on your machine or isn't available for your + platform. (Please file a bug report!) + + +### Is it really faster than everything else? + +Generally, yes. A large number of benchmarks with detailed analysis for each is +[available on my blog](https://blog.burntsushi.net/ripgrep/). + +Summarizing, ripgrep is fast because: + +* It is built on top of + [Rust's regex engine](https://github.com/rust-lang/regex). + Rust's regex engine uses finite automata, SIMD and aggressive literal + optimizations to make searching very fast. (PCRE2 support can be opted into + with the `-P/--pcre2` flag.) +* Rust's regex library maintains performance with full Unicode support by + building UTF-8 decoding directly into its deterministic finite automaton + engine. +* It supports searching with either memory maps or by searching incrementally + with an intermediate buffer. The former is better for single files and the + latter is better for large directories. ripgrep chooses the best searching + strategy for you automatically. +* Applies your ignore patterns in `.gitignore` files using a + [`RegexSet`](https://docs.rs/regex/1/regex/struct.RegexSet.html). + That means a single file path can be matched against multiple glob patterns + simultaneously. +* It uses a lock-free parallel recursive directory iterator, courtesy of + [`crossbeam`](https://docs.rs/crossbeam) and + [`ignore`](https://docs.rs/ignore). + + +### Feature comparison + +Andy Lester, author of [ack](https://beyondgrep.com/), has published an +excellent table comparing the features of ack, ag, git-grep, GNU grep and +ripgrep: https://beyondgrep.com/feature-comparison/ + +Note that ripgrep has grown a few significant new features recently that +are not yet present in Andy's table. This includes, but is not limited to, +configuration files, passthru, support for searching compressed files, +multiline search and opt-in fancy regex support via PCRE2. + + +### Playground + +If you'd like to try ripgrep before installing, there's an unofficial +[playground](https://codapi.org/ripgrep/) and an [interactive +tutorial](https://codapi.org/try/ripgrep/). + +If you have any questions about these, please open an issue in the [tutorial +repo](https://github.com/nalgeon/tryxinyminutes). + + +### Installation + +The binary name for ripgrep is `rg`. + +**[Archives of precompiled binaries for ripgrep are available for Windows, +macOS and Linux.](https://github.com/BurntSushi/ripgrep/releases)** Linux and +Windows binaries are static executables. Users of platforms not explicitly +mentioned below are advised to download one of these archives. + +If you're a **macOS Homebrew** or a **Linuxbrew** user, then you can install +ripgrep from homebrew-core: + +``` +$ brew install ripgrep +``` + +If you're a **MacPorts** user, then you can install ripgrep from the +[official ports](https://www.macports.org/ports.php?by=name&substr=ripgrep): + +``` +$ sudo port install ripgrep +``` + +If you're a **Windows Chocolatey** user, then you can install ripgrep from the +[official repo](https://chocolatey.org/packages/ripgrep): + +``` +$ choco install ripgrep +``` + +If you're a **Windows Scoop** user, then you can install ripgrep from the +[official bucket](https://github.com/ScoopInstaller/Main/blob/master/bucket/ripgrep.json): + +``` +$ scoop install ripgrep +``` + +If you're a **Windows Winget** user, then you can install ripgrep from the +[winget-pkgs](https://github.com/microsoft/winget-pkgs/tree/master/manifests/b/BurntSushi/ripgrep) +repository: + +``` +$ winget install BurntSushi.ripgrep.MSVC +``` + +If you're an **Arch Linux** user, then you can install ripgrep from the official repos: + +``` +$ sudo pacman -S ripgrep +``` + +If you're a **Gentoo** user, you can install ripgrep from the +[official repo](https://packages.gentoo.org/packages/sys-apps/ripgrep): + +``` +$ sudo emerge sys-apps/ripgrep +``` + +If you're a **Fedora** user, you can install ripgrep from official +repositories. + +``` +$ sudo dnf install ripgrep +``` + +If you're an **openSUSE** user, ripgrep is included in **openSUSE Tumbleweed** +and **openSUSE Leap** since 15.1. + +``` +$ sudo zypper install ripgrep +``` + +If you're a **CentOS Stream 10** user, you can install ripgrep from the +[EPEL](https://docs.fedoraproject.org/en-US/epel/getting-started/) repository: + +``` +$ sudo dnf config-manager --set-enabled crb +$ sudo dnf install https://dl.fedoraproject.org/pub/epel/epel-release-latest-10.noarch.rpm +$ sudo dnf install ripgrep +``` + +If you're a **Red Hat 10** user, you can install ripgrep from the +[EPEL](https://docs.fedoraproject.org/en-US/epel/getting-started/) repository: + +``` +$ sudo subscription-manager repos --enable codeready-builder-for-rhel-10-$(arch)-rpms +$ sudo dnf install https://dl.fedoraproject.org/pub/epel/epel-release-latest-10.noarch.rpm +$ sudo dnf install ripgrep +``` + +If you're a **Rocky Linux 10** user, you can install ripgrep from the +[EPEL](https://docs.fedoraproject.org/en-US/epel/getting-started/) repository: + +``` +$ sudo dnf install https://dl.fedoraproject.org/pub/epel/epel-release-latest-10.noarch.rpm +$ sudo dnf install ripgrep +``` + +If you're a **Nix** user, you can install ripgrep from +[nixpkgs](https://github.com/NixOS/nixpkgs/blob/master/pkgs/by-name/ri/ripgrep/package.nix): + +``` +$ nix-env --install ripgrep +``` + +If you're a **Flox** user, you can install ripgrep as follows: + +``` +$ flox install ripgrep +``` + +If you're a **Guix** user, you can install ripgrep from the official +package collection: + +``` +$ guix install ripgrep +``` + +If you're a **Debian** user (or a user of a Debian derivative like **Ubuntu**), +then ripgrep can be installed using a binary `.deb` file provided in each +[ripgrep release](https://github.com/BurntSushi/ripgrep/releases). + +``` +$ curl -LO https://github.com/BurntSushi/ripgrep/releases/download/14.1.1/ripgrep_14.1.1-1_amd64.deb +$ sudo dpkg -i ripgrep_14.1.1-1_amd64.deb +``` + +If you run Debian stable, ripgrep is [officially maintained by +Debian](https://tracker.debian.org/pkg/rust-ripgrep), although its version may +be older than the `deb` package available in the previous step. + +``` +$ sudo apt-get install ripgrep +``` + +If you're an **Ubuntu Cosmic (18.10)** (or newer) user, ripgrep is +[available](https://launchpad.net/ubuntu/+source/rust-ripgrep) using the same +packaging as Debian: + +``` +$ sudo apt-get install ripgrep +``` + +(N.B. Various snaps for ripgrep on Ubuntu are also available, but none of them +seem to work right and generate a number of very strange bug reports that I +don't know how to fix and don't have the time to fix. Therefore, it is no +longer a recommended installation option.) + +If you're an **ALT** user, you can install ripgrep from the +[official repo](https://packages.altlinux.org/en/search?name=ripgrep): + +``` +$ sudo apt-get install ripgrep +``` + +If you're a **FreeBSD** user, then you can install ripgrep from the +[official ports](https://www.freshports.org/textproc/ripgrep/): + +``` +$ sudo pkg install ripgrep +``` + +If you're an **OpenBSD** user, then you can install ripgrep from the +[official ports](https://openports.se/textproc/ripgrep): + +``` +$ doas pkg_add ripgrep +``` + +If you're a **NetBSD** user, then you can install ripgrep from +[pkgsrc](https://pkgsrc.se/textproc/ripgrep): + +``` +$ sudo pkgin install ripgrep +``` + +If you're a **Haiku x86_64** user, then you can install ripgrep from the +[official ports](https://github.com/haikuports/haikuports/tree/master/sys-apps/ripgrep): + +``` +$ sudo pkgman install ripgrep +``` + +If you're a **Haiku x86_gcc2** user, then you can install ripgrep from the +same port as Haiku x86_64 using the x86 secondary architecture build: + +``` +$ sudo pkgman install ripgrep_x86 +``` + +If you're a **Void Linux** user, then you can install ripgrep from the +[official repository](https://voidlinux.org/packages/?arch=x86_64&q=ripgrep): + +``` +$ sudo xbps-install -Syv ripgrep +``` + +If you're a **Rust programmer**, ripgrep can be installed with `cargo`. + +* Note that the minimum supported version of Rust for ripgrep is **1.85.0**, + although ripgrep may work with older versions. +* Note that the binary may be bigger than expected because it contains debug + symbols. This is intentional. To remove debug symbols and therefore reduce + the file size, run `strip` on the binary. + +``` +$ cargo install ripgrep +``` + +Alternatively, one can use [`cargo +binstall`](https://github.com/cargo-bins/cargo-binstall) to install a ripgrep +binary directly from GitHub: + +``` +$ cargo binstall ripgrep +``` + + +### Building + +ripgrep is written in Rust, so you'll need to grab a +[Rust installation](https://www.rust-lang.org/) in order to compile it. +ripgrep compiles with Rust 1.85.0 (stable) or newer. In general, ripgrep tracks +the latest stable release of the Rust compiler. + +To build ripgrep: + +``` +$ git clone https://github.com/BurntSushi/ripgrep +$ cd ripgrep +$ cargo build --release +$ ./target/release/rg --version +0.1.3 +``` + +**NOTE:** In the past, ripgrep supported a `simd-accel` Cargo feature when +using a Rust nightly compiler. This only benefited UTF-16 transcoding. +Since it required unstable features, this build mode was prone to breakage. +Because of that, support for it has been removed. If you want SIMD +optimizations for UTF-16 transcoding, then you'll have to petition the +[`encoding_rs`](https://github.com/hsivonen/encoding_rs) project to use stable +APIs. + +Finally, optional PCRE2 support can be built with ripgrep by enabling the +`pcre2` feature: + +``` +$ cargo build --release --features 'pcre2' +``` + +Enabling the PCRE2 feature works with a stable Rust compiler and will +attempt to automatically find and link with your system's PCRE2 library via +`pkg-config`. If one doesn't exist, then ripgrep will build PCRE2 from source +using your system's C compiler and then statically link it into the final +executable. Static linking can be forced even when there is an available PCRE2 +system library by either building ripgrep with the MUSL target or by setting +`PCRE2_SYS_STATIC=1`. + +ripgrep can be built with the MUSL target on Linux by first installing the MUSL +library on your system (consult your friendly neighborhood package manager). +Then you just need to add MUSL support to your Rust toolchain and rebuild +ripgrep, which yields a fully static executable: + +``` +$ rustup target add x86_64-unknown-linux-musl +$ cargo build --release --target x86_64-unknown-linux-musl +``` + +Applying the `--features` flag from above works as expected. If you want to +build a static executable with MUSL and with PCRE2, then you will need to have +`musl-gcc` installed, which might be in a separate package from the actual +MUSL library, depending on your Linux distribution. + + +### Running tests + +ripgrep is relatively well-tested, including both unit tests and integration +tests. To run the full test suite, use: + +``` +$ cargo test --all +``` + +from the repository root. + + +### Related tools + +* [delta](https://github.com/dandavison/delta) is a syntax highlighting +pager that supports the `rg --json` output format. So all you need to do to +make it work is `rg --json pattern | delta`. See [delta's manual section on +grep](https://dandavison.github.io/delta/grep.html) for more details. + + +### Vulnerability reporting + +For reporting a security vulnerability, please +[contact Andrew Gallant](https://blog.burntsushi.net/about/). +The contact page has my email address and PGP public key if you wish to send an +encrypted message. + + +### Translations + +The following is a list of known translations of ripgrep's documentation. These +are unofficially maintained and may not be up to date. + +* [Chinese](https://github.com/chinanf-boy/ripgrep-zh#%E6%9B%B4%E6%96%B0-) +* [Spanish](https://github.com/UltiRequiem/traducciones/tree/master/ripgrep) diff --git a/userland/ripgrep/RELEASE-CHECKLIST.md b/userland/ripgrep/RELEASE-CHECKLIST.md new file mode 100644 index 0000000..eb91916 --- /dev/null +++ b/userland/ripgrep/RELEASE-CHECKLIST.md @@ -0,0 +1,57 @@ +# Release Checklist + +* Ensure local `master` is up to date with respect to `origin/master`. +* Run `cargo update` and review dependency updates. Commit updated + `Cargo.lock`. +* Run `cargo outdated` and review semver incompatible updates. Unless there is + a strong motivation otherwise, review and update every dependency. Also + run `--aggressive`, but don't update to crates that are still in beta. +* Update date in `crates/core/flags/doc/template.rg.1`. +* Update the CHANGELOG as appropriate. +* Review changes for every crate in `crates` since the last ripgrep release. + If the set of changes is non-empty, issue a new release for that crate. Check + crates in the following order. After updating a crate, ensure minimal + versions are updated as appropriate in dependents. If an update is required, + run `cargo-up --no-push crates/{CRATE}/Cargo.toml`. + * crates/globset + * crates/ignore + * crates/cli + * crates/matcher + * crates/regex + * crates/pcre2 + * crates/searcher + * crates/printer + * crates/grep (bump minimal versions as necessary) + * crates/core (do **not** bump version, but update dependencies as needed) +* Edit the `Cargo.toml` to set the new ripgrep version. Run + `cargo update -p ripgrep` so that the `Cargo.lock` is updated. Commit the + changes and create a new signed tag. Alternatively, use + `cargo-up --no-push --no-release Cargo.toml {VERSION}` to automate this. +* Run `cargo package` and ensure it succeeds. +* Push changes to GitHub, NOT including the tag. (But do not publish a new + version of ripgrep to crates.io yet.) +* Once CI for `master` finishes successfully, push the version tag. (Trying to + do this in one step seems to result in GitHub Actions not seeing the tag + push and thus not running the release workflow.) +* Wait for CI to finish creating the release. If the release build fails, then + delete the tag from GitHub, make fixes, re-tag, delete the release and push. +* Copy the relevant section of the CHANGELOG to the tagged release notes. + Include this blurb describing what ripgrep is: + > In case you haven't heard of it before, ripgrep is a line-oriented search + > tool that recursively searches the current directory for a regex pattern. + > By default, ripgrep will respect gitignore rules and automatically skip + > hidden files/directories and binary files. +* Run `cargo publish`. +* Run `ci/sha256-releases {VERSION} >> pkg/brew/ripgrep-bin.rb`. Then edit + `pkg/brew/ripgrep-bin.rb` to update the version number and sha256 hashes. + Remove extraneous stuff added by `ci/sha256-releases`. Commit changes. +* Add TBD section to the top of the CHANGELOG: + ``` + TBD + === + Unreleased changes. Release notes have not yet been written. + ``` + +Note that [`cargo-up` can be found in BurntSushi's dotfiles][dotfiles]. + +[dotfiles]: https://github.com/BurntSushi/dotfiles/blob/master/bin/cargo-up diff --git a/userland/ripgrep/UNLICENSE b/userland/ripgrep/UNLICENSE new file mode 100644 index 0000000..68a49da --- /dev/null +++ b/userland/ripgrep/UNLICENSE @@ -0,0 +1,24 @@ +This is free and unencumbered software released into the public domain. + +Anyone is free to copy, modify, publish, use, compile, sell, or +distribute this software, either in source code form or as a compiled +binary, for any purpose, commercial or non-commercial, and by any +means. + +In jurisdictions that recognize copyright laws, the author or authors +of this software dedicate any and all copyright interest in the +software to the public domain. We make this dedication for the benefit +of the public at large and to the detriment of our heirs and +successors. We intend this dedication to be an overt act of +relinquishment in perpetuity of all present and future rights to this +software under copyright law. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +OTHER DEALINGS IN THE SOFTWARE. + +For more information, please refer to diff --git a/userland/ripgrep/build.rs b/userland/ripgrep/build.rs new file mode 100644 index 0000000..3479b69 --- /dev/null +++ b/userland/ripgrep/build.rs @@ -0,0 +1,61 @@ +fn main() { + set_git_revision_hash(); + set_windows_exe_options(); +} + +/// Embed a Windows manifest and set some linker options. +/// +/// The main reason for this is to enable long path support on Windows. This +/// still, I believe, requires enabling long path support in the registry. But +/// if that's enabled, then this will let ripgrep use C:\... style paths that +/// are longer than 260 characters. +fn set_windows_exe_options() { + static MANIFEST: &str = "pkg/windows/Manifest.xml"; + + let Ok(target_os) = std::env::var("CARGO_CFG_TARGET_OS") else { return }; + let Ok(target_env) = std::env::var("CARGO_CFG_TARGET_ENV") else { return }; + if !(target_os == "windows" && target_env == "msvc") { + return; + } + + let Ok(mut manifest) = std::env::current_dir() else { return }; + manifest.push(MANIFEST); + let Some(manifest) = manifest.to_str() else { return }; + + println!("cargo:rerun-if-changed={MANIFEST}"); + // Embed the Windows application manifest file. + println!("cargo:rustc-link-arg-bin=rg=/MANIFEST:EMBED"); + println!("cargo:rustc-link-arg-bin=rg=/MANIFESTINPUT:{manifest}"); + // Turn linker warnings into errors. Helps debugging, otherwise the + // warnings get squashed (I believe). + println!("cargo:rustc-link-arg-bin=rg=/WX"); +} + +/// Make the current git hash available to the build as the environment +/// variable `RIPGREP_BUILD_GIT_HASH`. +fn set_git_revision_hash() { + use std::process::Command; + + let args = &["rev-parse", "--short=10", "HEAD"]; + let output = Command::new("git").args(args).output(); + match output { + Ok(output) => { + let rev = + String::from_utf8_lossy(&output.stdout).trim().to_string(); + if rev.is_empty() { + println!( + "cargo:warning=output from `git rev-parse` is empty, \ + so skipping embedding of commit hash" + ); + return; + } + println!("cargo:rustc-env=RIPGREP_BUILD_GIT_HASH={rev}"); + } + Err(e) => { + println!( + "cargo:warning=failed to run `git rev-parse`, \ + so skipping embedding of commit hash: {e}" + ); + } + } +} diff --git a/userland/ripgrep/crates/core/README.md b/userland/ripgrep/crates/core/README.md new file mode 100644 index 0000000..44920e4 --- /dev/null +++ b/userland/ripgrep/crates/core/README.md @@ -0,0 +1,15 @@ +ripgrep core +------------ +This is the core ripgrep crate. In particular, `main.rs` is where the `main` +function lives. + +Most of ripgrep core consists of two things: + +* The definition of the CLI interface, including docs for every flag. +* Glue code that brings the `grep-matcher`, `grep-regex`, `grep-searcher` and + `grep-printer` crates together to actually execute the search. + +Currently, there are no plans to make ripgrep core available as an independent +library. However, much of the heavy lifting of ripgrep is done via its +constituent crates, which can be reused independent of ripgrep. Unfortunately, +there is no guide or tutorial to teach folks how to do this yet. diff --git a/userland/ripgrep/crates/core/flags/complete/bash.rs b/userland/ripgrep/crates/core/flags/complete/bash.rs new file mode 100644 index 0000000..a390061 --- /dev/null +++ b/userland/ripgrep/crates/core/flags/complete/bash.rs @@ -0,0 +1,107 @@ +/*! +Provides completions for ripgrep's CLI for the bash shell. +*/ + +use crate::flags::defs::FLAGS; + +const TEMPLATE_FULL: &'static str = " +_rg() { + local i cur prev opts cmds + COMPREPLY=() + cur=\"${COMP_WORDS[COMP_CWORD]}\" + prev=\"${COMP_WORDS[COMP_CWORD-1]}\" + cmd=\"\" + opts=\"\" + + for i in ${COMP_WORDS[@]}; do + case \"${i}\" in + rg) + cmd=\"rg\" + ;; + *) + ;; + esac + done + + case \"${cmd}\" in + rg) + opts=\"!OPTS!\" + if [[ ${cur} == -* || ${COMP_CWORD} -eq 1 ]] ; then + COMPREPLY=($(compgen -W \"${opts}\" -- \"${cur}\")) + return 0 + fi + case \"${prev}\" in +!CASES! + esac + COMPREPLY=($(compgen -W \"${opts}\" -- \"${cur}\")) + return 0 + ;; + esac +} + +complete -F _rg -o bashdefault -o default rg +"; + +const TEMPLATE_CASE: &'static str = " + !FLAG!) + COMPREPLY=($(compgen -f \"${cur}\")) + return 0 + ;; +"; + +const TEMPLATE_CASE_CHOICES: &'static str = " + !FLAG!) + COMPREPLY=($(compgen -W \"!CHOICES!\" -- \"${cur}\")) + return 0 + ;; +"; + +/// Generate completions for Bash. +/// +/// Note that these completions are based on what was produced for ripgrep <=13 +/// using Clap 2.x. Improvements on this are welcome. +pub(crate) fn generate() -> String { + let mut opts = String::new(); + for flag in FLAGS.iter() { + opts.push_str("--"); + opts.push_str(flag.name_long()); + opts.push(' '); + if let Some(short) = flag.name_short() { + opts.push('-'); + opts.push(char::from(short)); + opts.push(' '); + } + if let Some(name) = flag.name_negated() { + opts.push_str("--"); + opts.push_str(name); + opts.push(' '); + } + } + opts.push_str(" ..."); + + let mut cases = String::new(); + for flag in FLAGS.iter() { + let template = if !flag.doc_choices().is_empty() { + let choices = flag.doc_choices().join(" "); + TEMPLATE_CASE_CHOICES.trim_end().replace("!CHOICES!", &choices) + } else { + TEMPLATE_CASE.trim_end().to_string() + }; + let name = format!("--{}", flag.name_long()); + cases.push_str(&template.replace("!FLAG!", &name)); + if let Some(short) = flag.name_short() { + let name = format!("-{}", char::from(short)); + cases.push_str(&template.replace("!FLAG!", &name)); + } + if let Some(negated) = flag.name_negated() { + let name = format!("--{negated}"); + cases.push_str(&template.replace("!FLAG!", &name)); + } + } + + TEMPLATE_FULL + .replace("!OPTS!", &opts) + .replace("!CASES!", &cases) + .trim_start() + .to_string() +} diff --git a/userland/ripgrep/crates/core/flags/complete/encodings.sh b/userland/ripgrep/crates/core/flags/complete/encodings.sh new file mode 100644 index 0000000..14fc4a8 --- /dev/null +++ b/userland/ripgrep/crates/core/flags/complete/encodings.sh @@ -0,0 +1,29 @@ +# This is impossible to read, but these encodings rarely if ever change, so +# it probably does not matter. They are derived from the list given here: +# https://encoding.spec.whatwg.org/#concept-encoding-get +# +# The globbing here works in both fish and zsh (though they expand it in +# different orders). It may work in other shells too. + +{{,us-}ascii,arabic,chinese,cyrillic,greek{,8},hebrew,korean} +logical visual mac {,cs}macintosh x-mac-{cyrillic,roman,ukrainian} +866 ibm{819,866} csibm866 +big5{,-hkscs} {cn-,cs}big5 x-x-big5 +cp{819,866,125{0,1,2,3,4,5,6,7,8}} x-cp125{0,1,2,3,4,5,6,7,8} +csiso2022{jp,kr} csiso8859{6,8}{e,i} +csisolatin{1,2,3,4,5,6,9} csisolatin{arabic,cyrillic,greek,hebrew} +ecma-{114,118} asmo-708 elot_928 sun_eu_greek +euc-{jp,kr} x-euc-jp cseuckr cseucpkdfmtjapanese +{,x-}gbk csiso58gb231280 gb18030 {,cs}gb2312 gb_2312{,-80} hz-gb-2312 +iso-2022-{cn,cn-ext,jp,kr} +iso8859{,-}{1,2,3,4,5,6,7,8,9,10,11,13,14,15} +iso-8859-{1,2,3,4,5,6,7,8,9,10,11,{6,8}-{e,i},13,14,15,16} iso_8859-{1,2,3,4,5,6,7,8,9,15} +iso_8859-{1,2,6,7}:1987 iso_8859-{3,4,5,8}:1988 iso_8859-9:1989 +iso-ir-{58,100,101,109,110,126,127,138,144,148,149,157} +koi{,8,8-r,8-ru,8-u,8_r} cskoi8r +ks_c_5601-{1987,1989} ksc{,_}5691 csksc56011987 +latin{1,2,3,4,5,6} l{1,2,3,4,5,6,9} +shift{-,_}jis csshiftjis {,x-}sjis ms_kanji ms932 +utf{,-}8 utf-16{,be,le} unicode-1-1-utf-8 +windows-{31j,874,949,125{0,1,2,3,4,5,6,7,8}} dos-874 tis-620 ansi_x3.4-1968 +x-user-defined auto none diff --git a/userland/ripgrep/crates/core/flags/complete/fish.rs b/userland/ripgrep/crates/core/flags/complete/fish.rs new file mode 100644 index 0000000..a1ec2db --- /dev/null +++ b/userland/ripgrep/crates/core/flags/complete/fish.rs @@ -0,0 +1,75 @@ +/*! +Provides completions for ripgrep's CLI for the fish shell. +*/ + +use crate::flags::{CompletionType, defs::FLAGS}; + +const TEMPLATE: &'static str = "complete -c rg !SHORT! -l !LONG! -d '!DOC!'"; +const TEMPLATE_NEGATED: &'static str = "complete -c rg -l !NEGATED! -n '__rg_contains_opt !LONG! !SHORT!' -d '!DOC!'\n"; + +/// Generate completions for Fish. +/// +/// Reference: +pub(crate) fn generate() -> String { + let mut out = String::new(); + out.push_str(include_str!("prelude.fish")); + out.push('\n'); + for flag in FLAGS.iter() { + let short = match flag.name_short() { + None => "".to_string(), + Some(byte) => format!("-s {}", char::from(byte)), + }; + let long = flag.name_long(); + let doc = flag.doc_short().replace("'", "\\'"); + let mut completion = TEMPLATE + .replace("!SHORT!", &short) + .replace("!LONG!", &long) + .replace("!DOC!", &doc); + + match flag.completion_type() { + CompletionType::Filename => { + completion.push_str(" -r -F"); + } + CompletionType::Executable => { + completion.push_str(" -r -f -a '(__fish_complete_command)'"); + } + CompletionType::Filetype => { + completion.push_str( + " -r -f -a '(rg --type-list | string replace : \\t)'", + ); + } + CompletionType::Encoding => { + completion.push_str(" -r -f -a '"); + completion.push_str(super::ENCODINGS); + completion.push_str("'"); + } + CompletionType::Other if !flag.doc_choices().is_empty() => { + completion.push_str(" -r -f -a '"); + completion.push_str(&flag.doc_choices().join(" ")); + completion.push_str("'"); + } + CompletionType::Other if !flag.is_switch() => { + completion.push_str(" -r -f"); + } + CompletionType::Other => (), + } + + completion.push('\n'); + out.push_str(&completion); + + if let Some(negated) = flag.name_negated() { + let short = match flag.name_short() { + None => "".to_string(), + Some(byte) => char::from(byte).to_string(), + }; + out.push_str( + &TEMPLATE_NEGATED + .replace("!NEGATED!", &negated) + .replace("!SHORT!", &short) + .replace("!LONG!", &long) + .replace("!DOC!", &doc), + ); + } + } + out +} diff --git a/userland/ripgrep/crates/core/flags/complete/mod.rs b/userland/ripgrep/crates/core/flags/complete/mod.rs new file mode 100644 index 0000000..b531bf1 --- /dev/null +++ b/userland/ripgrep/crates/core/flags/complete/mod.rs @@ -0,0 +1,10 @@ +/*! +Modules for generating completions for various shells. +*/ + +static ENCODINGS: &'static str = include_str!("encodings.sh"); + +pub(super) mod bash; +pub(super) mod fish; +pub(super) mod powershell; +pub(super) mod zsh; diff --git a/userland/ripgrep/crates/core/flags/complete/powershell.rs b/userland/ripgrep/crates/core/flags/complete/powershell.rs new file mode 100644 index 0000000..bcd0e6f --- /dev/null +++ b/userland/ripgrep/crates/core/flags/complete/powershell.rs @@ -0,0 +1,85 @@ +/*! +Provides completions for ripgrep's CLI for PowerShell. +*/ + +use crate::flags::defs::FLAGS; + +const TEMPLATE: &'static str = " +using namespace System.Management.Automation +using namespace System.Management.Automation.Language + +Register-ArgumentCompleter -Native -CommandName 'rg' -ScriptBlock { + param($wordToComplete, $commandAst, $cursorPosition) + $commandElements = $commandAst.CommandElements + $command = @( + 'rg' + for ($i = 1; $i -lt $commandElements.Count; $i++) { + $element = $commandElements[$i] + if ($element -isnot [StringConstantExpressionAst] -or + $element.StringConstantType -ne [StringConstantType]::BareWord -or + $element.Value.StartsWith('-')) { + break + } + $element.Value + }) -join ';' + + $completions = @(switch ($command) { + 'rg' { +!FLAGS! + } + }) + + $completions.Where{ $_.CompletionText -like \"$wordToComplete*\" } | + Sort-Object -Property ListItemText +} +"; + +const TEMPLATE_FLAG: &'static str = "[CompletionResult]::new('!DASH_NAME!', '!NAME!', [CompletionResultType]::ParameterName, '!DOC!')"; + +/// Generate completions for PowerShell. +/// +/// Note that these completions are based on what was produced for ripgrep <=13 +/// using Clap 2.x. Improvements on this are welcome. +pub(crate) fn generate() -> String { + let mut flags = String::new(); + for (i, flag) in FLAGS.iter().enumerate() { + let doc = flag.doc_short().replace("'", "''"); + + let dash_name = format!("--{}", flag.name_long()); + let name = flag.name_long(); + if i > 0 { + flags.push('\n'); + } + flags.push_str(" "); + flags.push_str( + &TEMPLATE_FLAG + .replace("!DASH_NAME!", &dash_name) + .replace("!NAME!", &name) + .replace("!DOC!", &doc), + ); + + if let Some(byte) = flag.name_short() { + let dash_name = format!("-{}", char::from(byte)); + let name = char::from(byte).to_string(); + flags.push_str("\n "); + flags.push_str( + &TEMPLATE_FLAG + .replace("!DASH_NAME!", &dash_name) + .replace("!NAME!", &name) + .replace("!DOC!", &doc), + ); + } + + if let Some(negated) = flag.name_negated() { + let dash_name = format!("--{negated}"); + flags.push_str("\n "); + flags.push_str( + &TEMPLATE_FLAG + .replace("!DASH_NAME!", &dash_name) + .replace("!NAME!", &negated) + .replace("!DOC!", &doc), + ); + } + } + TEMPLATE.trim_start().replace("!FLAGS!", &flags) +} diff --git a/userland/ripgrep/crates/core/flags/complete/prelude.fish b/userland/ripgrep/crates/core/flags/complete/prelude.fish new file mode 100644 index 0000000..6932f3e --- /dev/null +++ b/userland/ripgrep/crates/core/flags/complete/prelude.fish @@ -0,0 +1,31 @@ +# Usage: __rg_contains_opt LONG [SHORT] +function __rg_contains_opt --description 'Specialized __fish_contains_opt' + # Cache the config file because this function is called many times per + # completion attempt. + # The cache will persist for the entire shell session (even if the + # variable or the file contents change). + if not set -q __rg_config + set -g __rg_config + if set -qx RIPGREP_CONFIG_PATH + set __rg_config ( + cat -- $RIPGREP_CONFIG_PATH 2>/dev/null \ + | string trim \ + | string match -rv '^$|^#' + ) + end + end + + set -l commandline (commandline -cpo) (commandline -ct) $__rg_config + + if contains -- "--$argv[1]" $commandline + return 0 + end + + if set -q argv[2] + if string match -qr -- "^-[^-]*$argv[2]" $commandline + return 0 + end + end + + return 1 +end diff --git a/userland/ripgrep/crates/core/flags/complete/rg.zsh b/userland/ripgrep/crates/core/flags/complete/rg.zsh new file mode 100644 index 0000000..aacaddd --- /dev/null +++ b/userland/ripgrep/crates/core/flags/complete/rg.zsh @@ -0,0 +1,705 @@ +#compdef rg + +## +# zsh completion function for ripgrep +# +# Run ci/test-complete after building to ensure that the options supported by +# this function stay in synch with the `rg` binary. +# +# For convenience, a completion reference guide is included at the bottom of +# this file. +# +# Originally based on code from the zsh-users project — see copyright notice +# below. + +_rg() { + local no='!' + local -a args + + # ripgrep has many options which negate the effect of a more common one — for + # example, `--no-column` to negate `--column`, and `--messages` to negate + # `--no-messages`. There are so many of these, and they're so infrequently + # used, that some users will probably find it irritating if they're completed + # indiscriminately, so let's not do that unless either the current prefix + # matches one of those negation options or the user has the `complete-all` + # style set. Note that this prefix check has to be updated manually to account + # for all of the potential negation options listed below! + if + # We also want to list all of these options during testing + [[ $_RG_COMPLETE_LIST_ARGS == (1|t*|y*) ]] || + # --[imnpru]* => --ignore*, --messages, --no-*, --pcre2-unicode, + # --require-git, --unicode + [[ $PREFIX$SUFFIX == --[imnpru]* ]] || + zstyle -t ":completion:${curcontext}:" complete-all + then + no= + fi + + args=( + + '(exclusive)' + '(: * -)'{-h,--help}'[display help information]' + '(: * -)'{-V,--version}'[display version information]' + '(: * -)'--pcre2-version'[print the version of PCRE2 used by ripgrep, if available]' + + + '(buffered)' + '--line-buffered[force line buffering]' + $no"--no-line-buffered[don't force line buffering]" + '--block-buffered[force block buffering]' + $no"--no-block-buffered[don't force block buffering]" + + + '(byte-offset)' + {-b,--byte-offset}'[show 0-based byte offset for each matching line]' + $no"--no-byte-offset[don't show byte offset for each matching line]" + + + '(case)' + {-i,--ignore-case}'[search case-insensitively]' + {-s,--case-sensitive}'[search case-sensitively]' + {-S,--smart-case}'[search case-insensitively if pattern is all lowercase]' + + + '(context-a)' + '(context-c)'{-A+,--after-context=}'[specify lines to show after each match]: :\ + _numbers -d0 -l0 "lines of context after match" + ' + + + '(context-b)' + '(context-c)'{-B+,--before-context=}'[specify lines to show before each match]: :\ + _numbers -d0 -l0 "lines of context before match" + ' + + + '(context-c)' + '(context-a context-b)'{-C+,--context=}'[specify lines to show before and after each match]: :\ + _numbers -d0 -l0 "lines of context before/after match" + ' + + + '(column)' + '--column[show column numbers for matches]' + $no"--no-column[don't show column numbers for matches]" + + + '(context-separator)' + '--context-separator=[specify string used to separate non-continuous context lines in output]:separator' + $no"--no-context-separator[don't print context separators]" + + + '(count)' + {-c,--count}'[only show count of matching lines for each file]' + '--count-matches[only show count of individual matches for each file]' + '--include-zero[include files with zero matches in summary]' + $no"--no-include-zero[don't include files with zero matches in summary]" + + + '(encoding)' + {-E+,--encoding=}'[specify text encoding of files to search]: :_rg_encodings' + $no'--no-encoding[use default text encoding]' + + + '(engine)' + '--engine=[specify regex engine to use]:regex engine:(( + default\:"default engine" + pcre2\:"identical to --pcre2" + auto\:"identical to --auto-hybrid-regex" + ))' + + + file + '(1)*'{-f+,--file=}'[specify file containing patterns to search for]: :_files' + + + '(file-match)' + '(stats)'{-l,--files-with-matches}'[only show names of files with matches]' + '(stats)--files-without-match[only show names of files without matches]' + + + '(file-name)' + {-H,--with-filename}'[show file name for matches]' + {-I,--no-filename}"[don't show file name for matches]" + + + '(file-system)' + "--one-file-system[don't descend into directories on other file systems]" + $no'--no-one-file-system[descend into directories on other file systems]' + + + '(fixed)' + {-F,--fixed-strings}'[treat pattern as literal string instead of regular expression]' + $no"--no-fixed-strings[don't treat pattern as literal string]" + + + '(follow)' + {-L,--follow}'[follow symlinks]' + $no"--no-follow[don't follow symlinks]" + + + glob + '*'{-g+,--glob=}'[include/exclude files matching specified glob]:glob pattern:_files' + '*--iglob=[include/exclude files matching specified case-insensitive glob]:glob pattern:_files' + + + '(glob-case-insensitive)' + '--glob-case-insensitive[treat -g/--glob patterns case insensitively]' + $no'--no-glob-case-insensitive[treat -g/--glob patterns case sensitively]' + + + '(heading)' + '(pretty-vimgrep)--heading[show matches grouped by file name]' + "(pretty-vimgrep)--no-heading[don't show matches grouped by file name]" + + + '(hidden)' + {-.,--hidden}'[search hidden files and directories]' + $no"--no-hidden[don't search hidden files and directories]" + + + '(hybrid)' + '--auto-hybrid-regex[DEPRECATED: dynamically use PCRE2 if necessary]' + $no"--no-auto-hybrid-regex[DEPRECATED: don't dynamically use PCRE2 if necessary]" + + + '(ignore)' + "(--no-ignore-global --no-ignore-parent --no-ignore-vcs --no-ignore-dot)--no-ignore[don't respect ignore files]" + $no'(--ignore-global --ignore-parent --ignore-vcs --ignore-dot)--ignore[respect ignore files]' + + + '(ignore-file-case-insensitive)' + '--ignore-file-case-insensitive[process ignore files case insensitively]' + $no'--no-ignore-file-case-insensitive[process ignore files case sensitively]' + + + '(ignore-exclude)' + "--no-ignore-exclude[don't respect local exclude (ignore) files]" + $no'--ignore-exclude[respect local exclude (ignore) files]' + + + '(ignore-global)' + "--no-ignore-global[don't respect global ignore files]" + $no'--ignore-global[respect global ignore files]' + + + '(ignore-parent)' + "--no-ignore-parent[don't respect ignore files in parent directories]" + $no'--ignore-parent[respect ignore files in parent directories]' + + + '(ignore-vcs)' + "--no-ignore-vcs[don't respect version control ignore files]" + $no'--ignore-vcs[respect version control ignore files]' + + + '(require-git)' + "--no-require-git[don't require git repository to respect gitignore rules]" + $no'--require-git[require git repository to respect gitignore rules]' + + + '(ignore-dot)' + "--no-ignore-dot[don't respect .ignore files]" + $no'--ignore-dot[respect .ignore files]' + + + '(ignore-files)' + "--no-ignore-files[don't respect --ignore-file flags]" + $no'--ignore-files[respect --ignore-file files]' + + + '(invert-match)' + {-v,--invert-match}'[invert matching]' + $no"--no-invert-match[don't invert matching]" + + + '(json)' + '--json[output results in JSON Lines format]' + $no"--no-json[don't output results in JSON Lines format]" + + + '(line-number)' + {-n,--line-number}'[show line numbers for matches]' + {-N,--no-line-number}"[don't show line numbers for matches]" + + + '(line-terminator)' + '--crlf[use CRLF as line terminator]' + $no"--no-crlf[don't use CRLF as line terminator]" + '(text)--null-data[use NUL as line terminator]' + + + '(max-columns-preview)' + '--max-columns-preview[show preview for long lines (with -M)]' + $no"--no-max-columns-preview[don't show preview for long lines (with -M)]" + + + '(max-depth)' + {-d,--max-depth,\!--maxdepth}'[specify max number of directories to descend into]: :\ + _numbers -dunlimited -l0 "max directories to descend into" + ' + + + '(messages)' + '(--no-ignore-messages)--no-messages[suppress some error messages]' + $no"--messages[don't suppress error messages affected by --no-messages]" + + + '(messages-ignore)' + "--no-ignore-messages[don't show ignore-file parse error messages]" + $no'--ignore-messages[show ignore-file parse error messages]' + + + '(mmap)' + '--mmap[search using memory maps when possible]' + "--no-mmap[don't search using memory maps]" + + + '(multiline)' + {-U,--multiline}'[permit matching across multiple lines]' + $no'(multiline-dotall)--no-multiline[restrict matches to at most one line each]' + + + '(multiline-dotall)' + '(--no-multiline)--multiline-dotall[allow "." to match newline (with -U)]' + $no"(--no-multiline)--no-multiline-dotall[don't allow \".\" to match newline (with -U)]" + + + '(only)' + {-o,--only-matching}'[show only matching part of each line]' + + + '(passthru)' + '(--vimgrep)--passthru[show both matching and non-matching lines]' + '!(--vimgrep)--passthrough' + + + '(pcre2)' + {-P,--pcre2}'[enable matching with PCRE2]' + $no'(pcre2-unicode)--no-pcre2[disable matching with PCRE2]' + + + '(pcre2-unicode)' + $no'(--no-pcre2 --no-pcre2-unicode)--pcre2-unicode[DEPRECATED: enable PCRE2 Unicode mode (with -P)]' + '(--no-pcre2 --pcre2-unicode)--no-pcre2-unicode[DEPRECATED: disable PCRE2 Unicode mode (with -P)]' + + + '(pre)' + '(-z --search-zip)--pre=[specify preprocessor utility]:preprocessor utility:_command_names -e' + $no'--no-pre[disable preprocessor utility]' + + + '(pretty-vimgrep)' + '(heading)'{-p,--pretty}'[alias for --color=always --heading -n]' + '(heading passthru)--vimgrep[show results in vim-compatible format]' + + + regexp + '(1 file)*'{-e+,--regexp=}'[specify search pattern]:pattern' + + + '(replace)' + {-r+,--replace=}'[specify string used to replace matches]:replace string' + + + '(sort)' + '(threads)--sort=[sort results in ascending order (disables parallelism)]:sort method:(( + none\:"no sorting" + path\:"sort by file path" + modified\:"sort by last modified time" + accessed\:"sort by last accessed time" + created\:"sort by creation time" + ))' + '(threads)--sortr=[sort results in descending order (disables parallelism)]:sort method:(( + none\:"no sorting" + path\:"sort by file path" + modified\:"sort by last modified time" + accessed\:"sort by last accessed time" + created\:"sort by creation time" + ))' + '(threads)--sort-files[DEPRECATED: sort results by file path (disables parallelism)]' + $no"--no-sort-files[DEPRECATED: don't sort results]" + + + '(stats)' + '(--files file-match)--stats[show search statistics]' + $no"--no-stats[don't show search statistics]" + + + '(text)' + {-a,--text}'[search binary files as if they were text]' + "--binary[search binary files, don't print binary data]" + $no"--no-binary[don't search binary files]" + $no"(--null-data)--no-text[don't search binary files as if they were text]" + + + '(threads)' + '(sort)'{-j+,--threads=}'[specify approximate number of threads to use]: :\ + _numbers -d0 -l0 "max threads" + ' + + + '(trim)' + '--trim[trim any ASCII whitespace prefix from each line]' + $no"--no-trim[don't trim ASCII whitespace prefix from each line]" + + + type + '*'{-t+,--type=}'[only search files matching specified type]: :_rg_types' + '*--type-add=[add new glob for specified file type]: :_rg_type_specs' + '*--type-clear=[clear globs previously defined for specified file type]: :_rg_types' + # This should actually be exclusive with everything but other type options + '(: *)--type-list[show all supported file types and their associated globs]' + '*'{-T+,--type-not=}"[don't search files matching specified file type]: :_rg_types" + + + '(word-line)' + {-w,--word-regexp}'[only show matches surrounded by word boundaries]' + {-x,--line-regexp}'[only show matches surrounded by line boundaries]' + + + '(unicode)' + $no'--unicode[enable Unicode mode]' + '--no-unicode[disable Unicode mode]' + + + '(zip)' + '(--pre)'{-z,--search-zip}'[search in compressed files]' + $no"--no-search-zip[don't search in compressed files]" + + + misc + '--color=[specify when to use colors in output]:when to use colors:(( + never\:"never use colors" + auto\:"use colors or not based on stdout, TERM, etc." + always\:"always use colors" + ansi\:"always use ANSI colors (even on Windows)" + ))' + '*--colors=[specify color and style settings]: :_rg_color_specs' + '--debug[show debug messages]' + '--field-context-separator[set string to delimit fields in context lines]' + '--field-match-separator[set string to delimit fields in matching lines]' + '--generate=[generate specified data (e.g. man page or completion script)]:data to generate:(( + man\:"man page" + complete-bash\:"shell completions for bash" + complete-zsh\:"shell completions for zsh" + complete-fish\:"shell completions for fish" + complete-powershell\:"shell completions for PowerShell" + ))' + '--hostname-bin=[specify executable for getting system hostname]:hostname executable:_command_names -e' + '--hyperlink-format=[specify pattern for hyperlinks]: :_rg_hyperlink_formats' + '--trace[show more verbose debug messages]' + '--dfa-size-limit=[specify upper size limit of generated DFA]: :\ + _numbers -ubytes "max DFA size" K M G + ' + "(1 stats)--files[show each file that would be searched (but don't search)]" + '*--ignore-file=[specify additional ignore file]:ignore file:_files' + '(-M --max-columns)'{-M+,--max-columns=}'[specify max length of lines to print]: :\ + _numbers -ubytes -d0 -l0 "max line length" + ' + '(-m --max-count)'{-m+,--max-count=}'[specify max number of matches per file]: :\ + _numbers -dunlimited -l0 "max matches per file" + ' + '--max-filesize=[specify size above which files should be ignored]: :\ + _numbers -ubytes -dunlimited -l0 "max file size" K M G + ' + "--no-config[don't load configuration files]" + '(-0 --null)'{-0,--null}'[print NUL byte after file names]' + '--path-separator=[specify path separator to use when printing file names]:separator' + '*--pre-glob=[include/exclude files matching specified glob for preprocessing (with --pre)]:glob pattern:_files' + '(-q --quiet)'{-q,--quiet}'[suppress normal output]' + '--regex-size-limit=[specify upper size limit of compiled regex]: :\ + _numbers -ubytes "max compiled regex size" K M G + ' + '*'{-u,--unrestricted}'[reduce level of "smart" searching]' + '--stop-on-nonmatch[stop on first non-matching line after a matching one]' + + + operand + '(--files --type-list file regexp)1: :_guard "^-*" "search pattern"' + '(--type-list)*:file or directory to search:_files' + ) + + # This is used with test-complete to verify that there are no options + # listed in the help output that aren't also defined here + [[ $_RG_COMPLETE_LIST_ARGS == (1|t*|y*) ]] && { + print -rl - $args + return 0 + } + + _arguments -s -S : $args +} + +# Complete colour specs +(( $+functions[_rg_color_specs] )) || +_rg_color_specs() { + local tag=color-specs + + if compset -P '[^:]##:(bg|fg):'; then + _describe -t $tag 'color name or r,g,b' '( + black blue green red cyan magenta yellow white + )' + + elif compset -P '[^:]##:style:'; then + _describe -t $tag 'style name' '( + {,no}bold {,no}intense {,no}underline + )' + + elif compset -P '[^:]##:'; then + _describe -t $tag 'color/style attribute' \ + '( + bg:"specify background color" + fg:"specify foreground color" + style:"specify text style" + )' -r: -S: \ + -- \ + '( none:"clear color/style for type" )' + + else + _describe -t $tag 'color/style type' '( + column:"specify coloring for column numbers" + line:"specify coloring for line numbers" + match:"specify coloring for match text" + highlight:"specify coloring for matching lines" + path:"specify coloring for file names" + )' -r: -S: + fi +} + +# Complete type specs +(( $+functions[_rg_type_specs] )) || +_rg_type_specs() { + local ret=1 + local -a expl + + if compset -P '[^:]##:include:'; then + _sequence -s , _rg_types && ret=0 + + elif compset -P '[^:]##:'; then + _describe -t type-specs-include 'include directive' '( + include:"include other type definitions" + )' -r: -S: && ret=0 + _wanted type-specs-glob expl 'glob pattern' _files && ret=0 + + else + _wanted type-specs-name expl 'file type (or new name)' \ + _rg_types -r: -S: && ret=0 + fi + + return ret +} + +# Complete encodings +(( $+functions[_rg_encodings] )) || +_rg_encodings() { + local -aU encodings=( +!ENCODINGS! + ) + encodings=( ${encodings//:/\\:} ) + _describe -t encodings encoding encodings "$@" +} + +# Complete file types +(( $+functions[_rg_types] )) || +_rg_types() { + local -aU types=( + # this assumes none of the patterns contain colons + ${(@)${(f)"$( _call_program types $words[1] --type-list )"}//:[[:space:]]##/:} + ) + + zstyle -t ":completion:${curcontext}:types" extra-verbose || + types=( ${types%%:*} ) + + _describe -t types 'file type' types "$@" +} + +# Complete hyperlink format-string aliases +(( $+functions[_rg_hyperlink_format_aliases] )) || +_rg_hyperlink_format_aliases() { + _describe -t format-aliases 'hyperlink format alias' '( +!HYPERLINK_ALIASES! + )' "$@" +} + +# Complete custom hyperlink format strings +(( $+functions[_rg_hyperlink_format_strings] )) || +_rg_hyperlink_format_strings() { + local bs + local -a pfx sfx rmv + + [[ -n $compstate[quote] ]] || bs=\\ + + sfx=( -S "$bs}" ) + rmv=( -r '}\-' ) + + compset -S "$bs{*" + compset -S "$bs}*" && sfx=( -S '' ) + compset -P '*}' + compset -P '[^{}]##' + compset -P '*{' || pfx=( -P "$bs{" ) + + WSL_DISTRO_NAME=${WSL_DISTRO_NAME:-\$WSL_DISTRO_NAME} \ + _describe -t format-variables 'hyperlink format variable' '( + path:"absolute path to file containing match (required)" + host:"system host name or output of --hostname-bin executable" + line:"line number of match" + column:"column of match (requires {line})" + wslprefix:"\"wsl$/$WSL_DISTRO_NAME\" (for WSL share)" + )' "${(@)pfx}" "${(@)sfx}" "${(@)rmv}" +} + +# Complete hyperlink formats +(( $+functions[_rg_hyperlink_formats] )) || +_rg_hyperlink_formats() { + _alternative \ + 'format-string-aliases: :_rg_hyperlink_format_aliases' \ + 'format-strings: :_rg_hyperlink_format_strings' +} + +# Don't run the completion function when being sourced by itself. +# +# See https://github.com/BurntSushi/ripgrep/issues/2956 +# See https://github.com/BurntSushi/ripgrep/pull/2957 +if [[ $funcstack[1] == _rg ]] || (( ! $+functions[compdef] )); then + _rg "$@" +else + compdef _rg rg +fi + +################################################################################ +# ZSH COMPLETION REFERENCE +# +# For the convenience of developers who aren't especially familiar with zsh +# completion functions, a brief reference guide follows. This is in no way +# comprehensive; it covers just enough of the basic structure, syntax, and +# conventions to help someone make simple changes like adding new options. For +# more complete documentation regarding zsh completion functions, please see the +# following: +# +# * http://zsh.sourceforge.net/Doc/Release/Completion-System.html +# * https://github.com/zsh-users/zsh/blob/master/Etc/completion-style-guide +# +# OVERVIEW +# +# Most zsh completion functions are defined in terms of `_arguments`, which is a +# shell function that takes a series of argument specifications. The specs for +# `rg` are stored in an array, which is common for more complex functions; the +# elements of the array are passed to `_arguments` on invocation. +# +# ARGUMENT-SPECIFICATION SYNTAX +# +# The following is a contrived example of the argument specs for a simple tool: +# +# '(: * -)'{-h,--help}'[display help information]' +# '(-q -v --quiet --verbose)'{-q,--quiet}'[decrease output verbosity]' +# '!(-q -v --quiet --verbose)--silent' +# '(-q -v --quiet --verbose)'{-v,--verbose}'[increase output verbosity]' +# '--color=[specify when to use colors]:when:(always never auto)' +# '*:example file:_files' +# +# Although there may appear to be six specs here, there are actually nine; we +# use brace expansion to combine specs for options that go by multiple names, +# like `-q` and `--quiet`. This is customary, and ties in with the fact that zsh +# merges completion possibilities together when they have the same description. +# +# The first line defines the option `-h`/`--help`. With most tools, it isn't +# useful to complete anything after `--help` because it effectively overrides +# all others; the `(: * -)` at the beginning of the spec tells zsh not to +# complete any other operands (`:` and `*`) or options (`-`) after this one has +# been used. The `[...]` at the end associates a description with `-h`/`--help`; +# as mentioned, zsh will see the identical descriptions and merge these options +# together when offering completion possibilities. +# +# The next line defines `-q`/`--quiet`. Here we don't want to suppress further +# completions entirely, but we don't want to offer `-q` if `--quiet` has been +# given (since they do the same thing), nor do we want to offer `-v` (since it +# doesn't make sense to be quiet and verbose at the same time). We don't need to +# tell zsh not to offer `--quiet` a second time, since that's the default +# behaviour, but since this line expands to two specs describing `-q` *and* +# `--quiet` we do need to explicitly list all of them here. +# +# The next line defines a hidden option `--silent` — maybe it's a deprecated +# synonym for `--quiet`. The leading `!` indicates that zsh shouldn't offer this +# option during completion. The benefit of providing a spec for an option that +# shouldn't be completed is that, if someone *does* use it, we can correctly +# suppress completion of other options afterwards. +# +# The next line defines `-v`/`--verbose`; this works just like `-q`/`--quiet`. +# +# The next line defines `--color`. In this example, `--color` doesn't have a +# corresponding short option, so we don't need to use brace expansion. Further, +# there are no other options it's exclusive with (just itself), so we don't need +# to define those at the beginning. However, it does take a mandatory argument. +# The `=` at the end of `--color=` indicates that the argument may appear either +# like `--color always` or like `--color=always`; this is how most GNU-style +# command-line tools work. The corresponding short option would normally use `+` +# — for example, `-c+` would allow either `-c always` or `-calways`. For this +# option, the arguments are known ahead of time, so we can simply list them in +# parentheses at the end (`when` is used as the description for the argument). +# +# The last line defines an operand (a non-option argument). In this example, the +# operand can be used any number of times (the leading `*`), and it should be a +# file path, so we tell zsh to call the `_files` function to complete it. The +# `example file` in the middle is the description to use for this operand; we +# could use a space instead to accept the default provided by `_files`. +# +# GROUPING ARGUMENT SPECIFICATIONS +# +# Newer versions of zsh support grouping argument specs together. All specs +# following a `+` and then a group name are considered to be members of the +# named group. Grouping is useful mostly for organisational purposes; it makes +# the relationship between different options more obvious, and makes it easier +# to specify exclusions. +# +# We could rewrite our example above using grouping as follows: +# +# '(: * -)'{-h,--help}'[display help information]' +# '--color=[specify when to use colors]:when:(always never auto)' +# '*:example file:_files' +# + '(verbosity)' +# {-q,--quiet}'[decrease output verbosity]' +# '!--silent' +# {-v,--verbose}'[increase output verbosity]' +# +# Here we take advantage of a useful feature of spec grouping — when the group +# name is surrounded by parentheses, as in `(verbosity)`, it tells zsh that all +# of the options in that group are exclusive with each other. As a result, we +# don't need to manually list out the exclusions at the beginning of each +# option. +# +# Groups can also be referred to by name in other argument specs; for example: +# +# '(xyz)--aaa' '*: :_files' +# + xyz --xxx --yyy --zzz +# +# Here we use the group name `xyz` to tell zsh that `--xxx`, `--yyy`, and +# `--zzz` are not to be completed after `--aaa`. This makes the exclusion list +# much more compact and reusable. +# +# CONVENTIONS +# +# zsh completion functions generally adhere to the following conventions: +# +# * Use two spaces for indentation +# * Combine specs for options with different names using brace expansion +# * In combined specs, list the short option first (as in `{-a,--text}`) +# * Use `+` or `=` as described above for options that take arguments +# * Provide a description for all options, option-arguments, and operands +# * Capitalise/punctuate argument descriptions as phrases, not complete +# sentences — 'display help information', never 'Display help information.' +# (but still capitalise acronyms and proper names) +# * Write argument descriptions as verb phrases — 'display x', 'enable y', +# 'use z' +# * Word descriptions to make it clear when an option expects an argument; +# usually this is done with the word 'specify', as in 'specify x' or +# 'use specified x') +# * Write argument descriptions as tersely as possible — for example, articles +# like 'a' and 'the' should be omitted unless it would be confusing +# +# Other conventions currently used by this function: +# +# * Order argument specs alphabetically by group name, then option name +# * Group options that are directly related, mutually exclusive, or frequently +# referenced by other argument specs +# * Use only characters in the set [a-z0-9_-] in group names +# * Order exclusion lists as follows: short options, long options, groups +# * Use American English in descriptions +# * Use 'don't' in descriptions instead of 'do not' +# * Word descriptions for related options as similarly as possible. For example, +# `--foo[enable foo]` and `--no-foo[disable foo]`, or `--foo[use foo]` and +# `--no-foo[don't use foo]` +# * Word descriptions to make it clear when an option only makes sense with +# another option, usually by adding '(with -x)' to the end +# * Don't quote strings or variables unnecessarily. When quotes are required, +# prefer single-quotes to double-quotes +# * Prefix option specs with `$no` when the option serves only to negate the +# behaviour of another option that must be provided explicitly by the user. +# This prevents rarely used options from cluttering up the completion menu +################################################################################ + +# ------------------------------------------------------------------------------ +# Copyright (c) 2011 Github zsh-users - http://github.com/zsh-users +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of the zsh-users nor the +# names of its contributors may be used to endorse or promote products +# derived from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL ZSH-USERS BE LIABLE FOR ANY +# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# ------------------------------------------------------------------------------ +# Description +# ----------- +# +# Completion script for ripgrep +# +# ------------------------------------------------------------------------------ +# Authors +# ------- +# +# * arcizan +# * MaskRay +# +# ------------------------------------------------------------------------------ + +# Local Variables: +# mode: shell-script +# coding: utf-8-unix +# indent-tabs-mode: nil +# sh-indentation: 2 +# sh-basic-offset: 2 +# End: +# vim: ft=zsh sw=2 ts=2 et diff --git a/userland/ripgrep/crates/core/flags/complete/zsh.rs b/userland/ripgrep/crates/core/flags/complete/zsh.rs new file mode 100644 index 0000000..0aa8b9c --- /dev/null +++ b/userland/ripgrep/crates/core/flags/complete/zsh.rs @@ -0,0 +1,32 @@ +/*! +Provides completions for ripgrep's CLI for the zsh shell. + +Unlike completion short for other shells (at time of writing), zsh's +completions for ripgrep are maintained by hand. This is because: + +1. They are lovingly written by an expert in such things. +2. Are much higher in quality than the ones below that are auto-generated. +Namely, the zsh completions take application level context about flag +compatibility into account. +3. There is a CI script that fails if a new flag is added to ripgrep that +isn't included in the zsh completions. +4. There is a wealth of documentation in the zsh script explaining how it +works and how it can be extended. + +In principle, I'd be open to maintaining any completion script by hand so +long as it meets criteria 3 and 4 above. +*/ + +/// Generate completions for zsh. +pub(crate) fn generate() -> String { + let hyperlink_alias_descriptions = grep::printer::hyperlink_aliases() + .iter() + .map(|alias| { + format!(r#" {}:"{}""#, alias.name(), alias.description()) + }) + .collect::>() + .join("\n"); + include_str!("rg.zsh") + .replace("!ENCODINGS!", super::ENCODINGS.trim_end()) + .replace("!HYPERLINK_ALIASES!", &hyperlink_alias_descriptions) +} diff --git a/userland/ripgrep/crates/core/flags/config.rs b/userland/ripgrep/crates/core/flags/config.rs new file mode 100644 index 0000000..c49716e --- /dev/null +++ b/userland/ripgrep/crates/core/flags/config.rs @@ -0,0 +1,176 @@ +/*! +This module provides routines for reading ripgrep config "rc" files. + +The primary output of these routines is a sequence of arguments, where each +argument corresponds precisely to one shell argument. +*/ + +use std::{ + ffi::OsString, + path::{Path, PathBuf}, +}; + +use bstr::{ByteSlice, io::BufReadExt}; + +/// Return a sequence of arguments derived from ripgrep rc configuration files. +pub fn args() -> Vec { + let config_path = match std::env::var_os("RIPGREP_CONFIG_PATH") { + None => { + log::debug!( + "RIPGREP_CONFIG_PATH environment variable is not set, \ + therefore not reading any config file" + ); + return vec![]; + } + Some(config_path) => { + if config_path.is_empty() { + return vec![]; + } + PathBuf::from(config_path) + } + }; + let (args, errs) = match parse(&config_path) { + Ok((args, errs)) => (args, errs), + Err(err) => { + message!( + "failed to read the file specified in RIPGREP_CONFIG_PATH: {}", + err + ); + return vec![]; + } + }; + if !errs.is_empty() { + for err in errs { + message!("{}:{}", config_path.display(), err); + } + } + log::debug!( + "{}: arguments loaded from config file: {:?}", + config_path.display(), + args + ); + args +} + +/// Parse a single ripgrep rc file from the given path. +/// +/// On success, this returns a set of shell arguments, in order, that should +/// be pre-pended to the arguments given to ripgrep at the command line. +/// +/// If the file could not be read, then an error is returned. If there was +/// a problem parsing one or more lines in the file, then errors are returned +/// for each line in addition to successfully parsed arguments. +fn parse>( + path: P, +) -> anyhow::Result<(Vec, Vec)> { + let path = path.as_ref(); + match std::fs::File::open(&path) { + Ok(file) => parse_reader(file), + Err(err) => anyhow::bail!("{}: {}", path.display(), err), + } +} + +/// Parse a single ripgrep rc file from the given reader. +/// +/// Callers should not provided a buffered reader, as this routine will use its +/// own buffer internally. +/// +/// On success, this returns a set of shell arguments, in order, that should +/// be pre-pended to the arguments given to ripgrep at the command line. +/// +/// If the reader could not be read, then an error is returned. If there was a +/// problem parsing one or more lines, then errors are returned for each line +/// in addition to successfully parsed arguments. +fn parse_reader( + rdr: R, +) -> anyhow::Result<(Vec, Vec)> { + let mut bufrdr = std::io::BufReader::new(rdr); + let (mut args, mut errs) = (vec![], vec![]); + let mut line_number = 0; + bufrdr.for_byte_line_with_terminator(|line| { + line_number += 1; + + let line = line.trim(); + if line.is_empty() || line[0] == b'#' { + return Ok(true); + } + match line.to_os_str() { + Ok(osstr) => { + args.push(osstr.to_os_string()); + } + Err(err) => { + errs.push(anyhow::anyhow!("{line_number}: {err}")); + } + } + Ok(true) + })?; + Ok((args, errs)) +} + +#[cfg(test)] +mod tests { + use super::parse_reader; + use std::ffi::OsString; + + #[test] + fn basic() { + let (args, errs) = parse_reader( + &b"\ +# Test +--context=0 + --smart-case +-u + + + # --bar +--foo +"[..], + ) + .unwrap(); + assert!(errs.is_empty()); + let args: Vec = + args.into_iter().map(|s| s.into_string().unwrap()).collect(); + assert_eq!(args, vec!["--context=0", "--smart-case", "-u", "--foo",]); + } + + // We test that we can handle invalid UTF-8 on Unix-like systems. + #[test] + #[cfg(unix)] + fn error() { + use std::os::unix::ffi::OsStringExt; + + let (args, errs) = parse_reader( + &b"\ +quux +foo\xFFbar +baz +"[..], + ) + .unwrap(); + assert!(errs.is_empty()); + assert_eq!( + args, + vec![ + OsString::from("quux"), + OsString::from_vec(b"foo\xFFbar".to_vec()), + OsString::from("baz"), + ] + ); + } + + // ... but test that invalid UTF-8 fails on Windows. + #[test] + #[cfg(not(unix))] + fn error() { + let (args, errs) = parse_reader( + &b"\ +quux +foo\xFFbar +baz +"[..], + ) + .unwrap(); + assert_eq!(errs.len(), 1); + assert_eq!(args, vec![OsString::from("quux"), OsString::from("baz"),]); + } +} diff --git a/userland/ripgrep/crates/core/flags/defs.rs b/userland/ripgrep/crates/core/flags/defs.rs new file mode 100644 index 0000000..87ac910 --- /dev/null +++ b/userland/ripgrep/crates/core/flags/defs.rs @@ -0,0 +1,7779 @@ +/*! +Defines all of the flags available in ripgrep. + +Each flag corresponds to a unit struct with a corresponding implementation +of `Flag`. Note that each implementation of `Flag` might actually have many +possible manifestations of the same "flag." That is, each implementation of +`Flag` can have the following flags available to an end user of ripgrep: + +* The long flag name. +* An optional short flag name. +* An optional negated long flag name. +* An arbitrarily long list of aliases. + +The idea is that even though there are multiple flags that a user can type, +one implementation of `Flag` corresponds to a single _logical_ flag inside of +ripgrep. For example, `-E`, `--encoding` and `--no-encoding` all manipulate the +same encoding state in ripgrep. +*/ + +use std::{path::PathBuf, sync::LazyLock}; + +use {anyhow::Context as AnyhowContext, bstr::ByteVec}; + +use crate::flags::{ + Category, Flag, FlagValue, + lowargs::{ + BinaryMode, BoundaryMode, BufferMode, CaseMode, ColorChoice, + ContextMode, EncodingMode, EngineChoice, GenerateMode, LoggingMode, + LowArgs, MmapMode, Mode, PatternSource, SearchMode, SortMode, + SortModeKind, SpecialMode, TypeChange, + }, +}; + +#[cfg(test)] +use crate::flags::parse::parse_low_raw; + +use super::CompletionType; + +/// A list of all flags in ripgrep via implementations of `Flag`. +/// +/// The order of these flags matter. It determines the order of the flags in +/// the generated documentation (`-h`, `--help` and the man page) within each +/// category. (This is why the deprecated flags are last.) +pub(super) const FLAGS: &[&dyn Flag] = &[ + // -e/--regexp and -f/--file should come before anything else in the + // same category. + &Regexp, + &File, + &AfterContext, + &BeforeContext, + &Binary, + &BlockBuffered, + &ByteOffset, + &CaseSensitive, + &Color, + &Colors, + &Column, + &Context, + &ContextSeparator, + &Count, + &CountMatches, + &Crlf, + &Debug, + &DfaSizeLimit, + &Encoding, + &Engine, + &FieldContextSeparator, + &FieldMatchSeparator, + &Files, + &FilesWithMatches, + &FilesWithoutMatch, + &FixedStrings, + &Follow, + &Generate, + &Glob, + &GlobCaseInsensitive, + &Heading, + &Help, + &Hidden, + &HostnameBin, + &HyperlinkFormat, + &IGlob, + &IgnoreCase, + &IgnoreFile, + &IgnoreFileCaseInsensitive, + &IncludeZero, + &InvertMatch, + &JSON, + &LineBuffered, + &LineNumber, + &LineNumberNo, + &LineRegexp, + &MaxColumns, + &MaxColumnsPreview, + &MaxCount, + &MaxDepth, + &MaxFilesize, + &Mmap, + &Multiline, + &MultilineDotall, + &NoConfig, + &NoIgnore, + &NoIgnoreDot, + &NoIgnoreExclude, + &NoIgnoreFiles, + &NoIgnoreGlobal, + &NoIgnoreMessages, + &NoIgnoreParent, + &NoIgnoreVcs, + &NoMessages, + &NoRequireGit, + &NoUnicode, + &Null, + &NullData, + &OneFileSystem, + &OnlyMatching, + &PathSeparator, + &Passthru, + &PCRE2, + &PCRE2Version, + &Pre, + &PreGlob, + &Pretty, + &Quiet, + &RegexSizeLimit, + &Replace, + &SearchZip, + &SmartCase, + &Sort, + &Sortr, + &Stats, + &StopOnNonmatch, + &Text, + &Threads, + &Trace, + &Trim, + &Type, + &TypeNot, + &TypeAdd, + &TypeClear, + &TypeList, + &Unrestricted, + &Version, + &Vimgrep, + &WithFilename, + &WithFilenameNo, + &WordRegexp, + // DEPRECATED (make them show up last in their respective categories) + &AutoHybridRegex, + &NoPcre2Unicode, + &SortFiles, +]; + +/// -A/--after-context +#[derive(Debug)] +struct AfterContext; + +impl Flag for AfterContext { + fn is_switch(&self) -> bool { + false + } + fn name_short(&self) -> Option { + Some(b'A') + } + fn name_long(&self) -> &'static str { + "after-context" + } + fn doc_variable(&self) -> Option<&'static str> { + Some("NUM") + } + fn doc_category(&self) -> Category { + Category::Output + } + fn doc_short(&self) -> &'static str { + "Show NUM lines after each match." + } + fn doc_long(&self) -> &'static str { + r" +Show \fINUM\fP lines after each match. +.sp +This overrides the \flag{passthru} flag and partially overrides the +\flag{context} flag. +" + } + + fn update(&self, v: FlagValue, args: &mut LowArgs) -> anyhow::Result<()> { + args.context.set_after(convert::usize(&v.unwrap_value())?); + Ok(()) + } +} + +#[cfg(test)] +#[test] +fn test_after_context() { + let mkctx = |lines| { + let mut mode = ContextMode::default(); + mode.set_after(lines); + mode + }; + + let args = parse_low_raw(None::<&str>).unwrap(); + assert_eq!(ContextMode::default(), args.context); + + let args = parse_low_raw(["--after-context", "5"]).unwrap(); + assert_eq!(mkctx(5), args.context); + + let args = parse_low_raw(["--after-context=5"]).unwrap(); + assert_eq!(mkctx(5), args.context); + + let args = parse_low_raw(["-A", "5"]).unwrap(); + assert_eq!(mkctx(5), args.context); + + let args = parse_low_raw(["-A5"]).unwrap(); + assert_eq!(mkctx(5), args.context); + + let args = parse_low_raw(["-A5", "-A10"]).unwrap(); + assert_eq!(mkctx(10), args.context); + + let args = parse_low_raw(["-A5", "-A0"]).unwrap(); + assert_eq!(mkctx(0), args.context); + + let args = parse_low_raw(["-A5", "--passthru"]).unwrap(); + assert_eq!(ContextMode::Passthru, args.context); + + let args = parse_low_raw(["--passthru", "-A5"]).unwrap(); + assert_eq!(mkctx(5), args.context); + + let n = usize::MAX.to_string(); + let args = parse_low_raw(["--after-context", n.as_str()]).unwrap(); + assert_eq!(mkctx(usize::MAX), args.context); + + #[cfg(target_pointer_width = "64")] + { + let n = (u128::from(u64::MAX) + 1).to_string(); + let result = parse_low_raw(["--after-context", n.as_str()]); + assert!(result.is_err(), "{result:?}"); + } +} + +/// --auto-hybrid-regex +#[derive(Debug)] +struct AutoHybridRegex; + +impl Flag for AutoHybridRegex { + fn is_switch(&self) -> bool { + true + } + fn name_long(&self) -> &'static str { + "auto-hybrid-regex" + } + fn name_negated(&self) -> Option<&'static str> { + Some("no-auto-hybrid-regex") + } + fn doc_category(&self) -> Category { + Category::Search + } + fn doc_short(&self) -> &'static str { + "(DEPRECATED) Use PCRE2 if appropriate." + } + fn doc_long(&self) -> &'static str { + r" +DEPRECATED. Use \flag{engine} instead. +.sp +When this flag is used, ripgrep will dynamically choose between supported regex +engines depending on the features used in a pattern. When ripgrep chooses a +regex engine, it applies that choice for every regex provided to ripgrep (e.g., +via multiple \flag{regexp} or \flag{file} flags). +.sp +As an example of how this flag might behave, ripgrep will attempt to use +its default finite automata based regex engine whenever the pattern can be +successfully compiled with that regex engine. If PCRE2 is enabled and if the +pattern given could not be compiled with the default regex engine, then PCRE2 +will be automatically used for searching. If PCRE2 isn't available, then this +flag has no effect because there is only one regex engine to choose from. +.sp +In the future, ripgrep may adjust its heuristics for how it decides which +regex engine to use. In general, the heuristics will be limited to a static +analysis of the patterns, and not to any specific runtime behavior observed +while searching files. +.sp +The primary downside of using this flag is that it may not always be obvious +which regex engine ripgrep uses, and thus, the match semantics or performance +profile of ripgrep may subtly and unexpectedly change. However, in many cases, +all regex engines will agree on what constitutes a match and it can be nice +to transparently support more advanced regex features like look-around and +backreferences without explicitly needing to enable them. +" + } + + fn update(&self, v: FlagValue, args: &mut LowArgs) -> anyhow::Result<()> { + let mode = if v.unwrap_switch() { + EngineChoice::Auto + } else { + EngineChoice::Default + }; + args.engine = mode; + Ok(()) + } +} + +#[cfg(test)] +#[test] +fn test_auto_hybrid_regex() { + let args = parse_low_raw(None::<&str>).unwrap(); + assert_eq!(EngineChoice::Default, args.engine); + + let args = parse_low_raw(["--auto-hybrid-regex"]).unwrap(); + assert_eq!(EngineChoice::Auto, args.engine); + + let args = + parse_low_raw(["--auto-hybrid-regex", "--no-auto-hybrid-regex"]) + .unwrap(); + assert_eq!(EngineChoice::Default, args.engine); + + let args = + parse_low_raw(["--no-auto-hybrid-regex", "--auto-hybrid-regex"]) + .unwrap(); + assert_eq!(EngineChoice::Auto, args.engine); + + let args = parse_low_raw(["--auto-hybrid-regex", "-P"]).unwrap(); + assert_eq!(EngineChoice::PCRE2, args.engine); + + let args = parse_low_raw(["-P", "--auto-hybrid-regex"]).unwrap(); + assert_eq!(EngineChoice::Auto, args.engine); + + let args = + parse_low_raw(["--engine=auto", "--auto-hybrid-regex"]).unwrap(); + assert_eq!(EngineChoice::Auto, args.engine); + + let args = + parse_low_raw(["--engine=default", "--auto-hybrid-regex"]).unwrap(); + assert_eq!(EngineChoice::Auto, args.engine); + + let args = + parse_low_raw(["--auto-hybrid-regex", "--engine=default"]).unwrap(); + assert_eq!(EngineChoice::Default, args.engine); +} + +/// -B/--before-context +#[derive(Debug)] +struct BeforeContext; + +impl Flag for BeforeContext { + fn is_switch(&self) -> bool { + false + } + fn name_short(&self) -> Option { + Some(b'B') + } + fn name_long(&self) -> &'static str { + "before-context" + } + fn doc_variable(&self) -> Option<&'static str> { + Some("NUM") + } + fn doc_category(&self) -> Category { + Category::Output + } + fn doc_short(&self) -> &'static str { + "Show NUM lines before each match." + } + fn doc_long(&self) -> &'static str { + r" +Show \fINUM\fP lines before each match. +.sp +This overrides the \flag{passthru} flag and partially overrides the +\flag{context} flag. +" + } + + fn update(&self, v: FlagValue, args: &mut LowArgs) -> anyhow::Result<()> { + args.context.set_before(convert::usize(&v.unwrap_value())?); + Ok(()) + } +} + +#[cfg(test)] +#[test] +fn test_before_context() { + let mkctx = |lines| { + let mut mode = ContextMode::default(); + mode.set_before(lines); + mode + }; + + let args = parse_low_raw(None::<&str>).unwrap(); + assert_eq!(ContextMode::default(), args.context); + + let args = parse_low_raw(["--before-context", "5"]).unwrap(); + assert_eq!(mkctx(5), args.context); + + let args = parse_low_raw(["--before-context=5"]).unwrap(); + assert_eq!(mkctx(5), args.context); + + let args = parse_low_raw(["-B", "5"]).unwrap(); + assert_eq!(mkctx(5), args.context); + + let args = parse_low_raw(["-B5"]).unwrap(); + assert_eq!(mkctx(5), args.context); + + let args = parse_low_raw(["-B5", "-B10"]).unwrap(); + assert_eq!(mkctx(10), args.context); + + let args = parse_low_raw(["-B5", "-B0"]).unwrap(); + assert_eq!(mkctx(0), args.context); + + let args = parse_low_raw(["-B5", "--passthru"]).unwrap(); + assert_eq!(ContextMode::Passthru, args.context); + + let args = parse_low_raw(["--passthru", "-B5"]).unwrap(); + assert_eq!(mkctx(5), args.context); + + let n = usize::MAX.to_string(); + let args = parse_low_raw(["--before-context", n.as_str()]).unwrap(); + assert_eq!(mkctx(usize::MAX), args.context); + + #[cfg(target_pointer_width = "64")] + { + let n = (u128::from(u64::MAX) + 1).to_string(); + let result = parse_low_raw(["--before-context", n.as_str()]); + assert!(result.is_err(), "{result:?}"); + } +} + +/// --binary +#[derive(Debug)] +struct Binary; + +impl Flag for Binary { + fn is_switch(&self) -> bool { + true + } + fn name_long(&self) -> &'static str { + "binary" + } + fn name_negated(&self) -> Option<&'static str> { + Some("no-binary") + } + fn doc_category(&self) -> Category { + Category::Filter + } + fn doc_short(&self) -> &'static str { + "Search binary files." + } + fn doc_long(&self) -> &'static str { + r" +Enabling this flag will cause ripgrep to search binary files. By default, +ripgrep attempts to automatically skip binary files in order to improve the +relevance of results and make the search faster. +.sp +Binary files are heuristically detected based on whether they contain a +\fBNUL\fP byte or not. By default (without this flag set), once a \fBNUL\fP +byte is seen, ripgrep will stop searching the file. Usually, \fBNUL\fP bytes +occur in the beginning of most binary files. If a \fBNUL\fP byte occurs after +a match, then ripgrep will not print the match, stop searching that file, and +emit a warning that some matches are being suppressed. +.sp +In contrast, when this flag is provided, ripgrep will continue searching a +file even if a \fBNUL\fP byte is found. In particular, if a \fBNUL\fP byte is +found then ripgrep will continue searching until either a match is found or +the end of the file is reached, whichever comes sooner. If a match is found, +then ripgrep will stop and print a warning saying that the search stopped +prematurely. +.sp +If you want ripgrep to search a file without any special \fBNUL\fP byte +handling at all (and potentially print binary data to stdout), then you should +use the \flag{text} flag. +.sp +The \flag{binary} flag is a flag for controlling ripgrep's automatic filtering +mechanism. As such, it does not need to be used when searching a file +explicitly or when searching stdin. That is, it is only applicable when +recursively searching a directory. +.sp +When the \flag{unrestricted} flag is provided for a third time, then this flag +is automatically enabled. +.sp +This flag overrides the \flag{text} flag. +" + } + + fn update(&self, v: FlagValue, args: &mut LowArgs) -> anyhow::Result<()> { + args.binary = if v.unwrap_switch() { + BinaryMode::SearchAndSuppress + } else { + BinaryMode::Auto + }; + Ok(()) + } +} + +#[cfg(test)] +#[test] +fn test_binary() { + let args = parse_low_raw(None::<&str>).unwrap(); + assert_eq!(BinaryMode::Auto, args.binary); + + let args = parse_low_raw(["--binary"]).unwrap(); + assert_eq!(BinaryMode::SearchAndSuppress, args.binary); + + let args = parse_low_raw(["--binary", "--no-binary"]).unwrap(); + assert_eq!(BinaryMode::Auto, args.binary); + + let args = parse_low_raw(["--no-binary", "--binary"]).unwrap(); + assert_eq!(BinaryMode::SearchAndSuppress, args.binary); + + let args = parse_low_raw(["--binary", "-a"]).unwrap(); + assert_eq!(BinaryMode::AsText, args.binary); + + let args = parse_low_raw(["-a", "--binary"]).unwrap(); + assert_eq!(BinaryMode::SearchAndSuppress, args.binary); + + let args = parse_low_raw(["-a", "--no-binary"]).unwrap(); + assert_eq!(BinaryMode::Auto, args.binary); +} + +/// --block-buffered +#[derive(Debug)] +struct BlockBuffered; + +impl Flag for BlockBuffered { + fn is_switch(&self) -> bool { + true + } + fn name_long(&self) -> &'static str { + "block-buffered" + } + fn name_negated(&self) -> Option<&'static str> { + Some("no-block-buffered") + } + fn doc_category(&self) -> Category { + Category::Output + } + fn doc_short(&self) -> &'static str { + "Force block buffering." + } + fn doc_long(&self) -> &'static str { + r" +When enabled, ripgrep will use block buffering. That is, whenever a matching +line is found, it will be written to an in-memory buffer and will not be +written to stdout until the buffer reaches a certain size. This is the default +when ripgrep's stdout is redirected to a pipeline or a file. When ripgrep's +stdout is connected to a tty, line buffering will be used by default. Forcing +block buffering can be useful when dumping a large amount of contents to a tty. +.sp +This overrides the \flag{line-buffered} flag. +" + } + + fn update(&self, v: FlagValue, args: &mut LowArgs) -> anyhow::Result<()> { + args.buffer = if v.unwrap_switch() { + BufferMode::Block + } else { + BufferMode::Auto + }; + Ok(()) + } +} + +#[cfg(test)] +#[test] +fn test_block_buffered() { + let args = parse_low_raw(None::<&str>).unwrap(); + assert_eq!(BufferMode::Auto, args.buffer); + + let args = parse_low_raw(["--block-buffered"]).unwrap(); + assert_eq!(BufferMode::Block, args.buffer); + + let args = + parse_low_raw(["--block-buffered", "--no-block-buffered"]).unwrap(); + assert_eq!(BufferMode::Auto, args.buffer); + + let args = parse_low_raw(["--block-buffered", "--line-buffered"]).unwrap(); + assert_eq!(BufferMode::Line, args.buffer); +} + +/// --byte-offset +#[derive(Debug)] +struct ByteOffset; + +impl Flag for ByteOffset { + fn is_switch(&self) -> bool { + true + } + fn name_short(&self) -> Option { + Some(b'b') + } + fn name_long(&self) -> &'static str { + "byte-offset" + } + fn name_negated(&self) -> Option<&'static str> { + Some("no-byte-offset") + } + fn doc_category(&self) -> Category { + Category::Output + } + fn doc_short(&self) -> &'static str { + "Print the byte offset for each matching line." + } + fn doc_long(&self) -> &'static str { + r" +Print the 0-based byte offset within the input file before each line of output. +If \flag{only-matching} is specified, print the offset of the matched text +itself. +.sp +If ripgrep does transcoding, then the byte offset is in terms of the result +of transcoding and not the original data. This applies similarly to other +transformations on the data, such as decompression or a \flag{pre} filter. +" + } + + fn update(&self, v: FlagValue, args: &mut LowArgs) -> anyhow::Result<()> { + args.byte_offset = v.unwrap_switch(); + Ok(()) + } +} + +#[cfg(test)] +#[test] +fn test_byte_offset() { + let args = parse_low_raw(None::<&str>).unwrap(); + assert_eq!(false, args.byte_offset); + + let args = parse_low_raw(["--byte-offset"]).unwrap(); + assert_eq!(true, args.byte_offset); + + let args = parse_low_raw(["-b"]).unwrap(); + assert_eq!(true, args.byte_offset); + + let args = parse_low_raw(["--byte-offset", "--no-byte-offset"]).unwrap(); + assert_eq!(false, args.byte_offset); + + let args = parse_low_raw(["--no-byte-offset", "-b"]).unwrap(); + assert_eq!(true, args.byte_offset); +} + +/// -s/--case-sensitive +#[derive(Debug)] +struct CaseSensitive; + +impl Flag for CaseSensitive { + fn is_switch(&self) -> bool { + true + } + fn name_short(&self) -> Option { + Some(b's') + } + fn name_long(&self) -> &'static str { + "case-sensitive" + } + fn doc_category(&self) -> Category { + Category::Search + } + fn doc_short(&self) -> &'static str { + r"Search case sensitively (default)." + } + fn doc_long(&self) -> &'static str { + r" +Execute the search case sensitively. This is the default mode. +.sp +This is a global option that applies to all patterns given to ripgrep. +Individual patterns can still be matched case insensitively by using inline +regex flags. For example, \fB(?i)abc\fP will match \fBabc\fP case insensitively +even when this flag is used. +.sp +This flag overrides the \flag{ignore-case} and \flag{smart-case} flags. +" + } + + fn update(&self, v: FlagValue, args: &mut LowArgs) -> anyhow::Result<()> { + assert!(v.unwrap_switch(), "flag has no negation"); + args.case = CaseMode::Sensitive; + Ok(()) + } +} + +#[cfg(test)] +#[test] +fn test_case_sensitive() { + let args = parse_low_raw(None::<&str>).unwrap(); + assert_eq!(CaseMode::Sensitive, args.case); + + let args = parse_low_raw(["--case-sensitive"]).unwrap(); + assert_eq!(CaseMode::Sensitive, args.case); + + let args = parse_low_raw(["-s"]).unwrap(); + assert_eq!(CaseMode::Sensitive, args.case); +} + +/// --color +#[derive(Debug)] +struct Color; + +impl Flag for Color { + fn is_switch(&self) -> bool { + false + } + fn name_long(&self) -> &'static str { + "color" + } + fn doc_variable(&self) -> Option<&'static str> { + Some("WHEN") + } + fn doc_category(&self) -> Category { + Category::Output + } + fn doc_short(&self) -> &'static str { + "When to use color." + } + fn doc_long(&self) -> &'static str { + r" +This flag controls when to use colors. The default setting is \fBauto\fP, which +means ripgrep will try to guess when to use colors. For example, if ripgrep is +printing to a tty, then it will use colors, but if it is redirected to a file +or a pipe, then it will suppress color output. +.sp +ripgrep will suppress color output by default in some other circumstances as +well. These include, but are not limited to: +.sp +.IP \(bu 3n +When the \fBTERM\fP environment variable is not set or set to \fBdumb\fP. +.sp +.IP \(bu 3n +When the \fBNO_COLOR\fP environment variable is set (regardless of value). +.sp +.IP \(bu 3n +When flags that imply no use for colors are given. For example, +\flag{vimgrep} and \flag{json}. +. +.PP +The possible values for this flag are: +.sp +.IP \fBnever\fP 10n +Colors will never be used. +.sp +.IP \fBauto\fP 10n +The default. ripgrep tries to be smart. +.sp +.IP \fBalways\fP 10n +Colors will always be used regardless of where output is sent. +.sp +.IP \fBansi\fP 10n +Like 'always', but emits ANSI escapes (even in a Windows console). +. +.PP +This flag also controls whether hyperlinks are emitted. For example, when +a hyperlink format is specified, hyperlinks won't be used when color is +suppressed. If one wants to emit hyperlinks but no colors, then one must use +the \flag{colors} flag to manually set all color styles to \fBnone\fP: +.sp +.EX + \-\-colors 'path:none' \\ + \-\-colors 'line:none' \\ + \-\-colors 'column:none' \\ + \-\-colors 'match:none' \\ + \-\-colors 'highlight:none' +.EE +.sp +" + } + fn doc_choices(&self) -> &'static [&'static str] { + &["never", "auto", "always", "ansi"] + } + + fn update(&self, v: FlagValue, args: &mut LowArgs) -> anyhow::Result<()> { + args.color = match convert::str(&v.unwrap_value())? { + "never" => ColorChoice::Never, + "auto" => ColorChoice::Auto, + "always" => ColorChoice::Always, + "ansi" => ColorChoice::Ansi, + unk => anyhow::bail!("choice '{unk}' is unrecognized"), + }; + Ok(()) + } +} + +#[cfg(test)] +#[test] +fn test_color() { + let args = parse_low_raw(None::<&str>).unwrap(); + assert_eq!(ColorChoice::Auto, args.color); + + let args = parse_low_raw(["--color", "never"]).unwrap(); + assert_eq!(ColorChoice::Never, args.color); + + let args = parse_low_raw(["--color", "auto"]).unwrap(); + assert_eq!(ColorChoice::Auto, args.color); + + let args = parse_low_raw(["--color", "always"]).unwrap(); + assert_eq!(ColorChoice::Always, args.color); + + let args = parse_low_raw(["--color", "ansi"]).unwrap(); + assert_eq!(ColorChoice::Ansi, args.color); + + let args = parse_low_raw(["--color=never"]).unwrap(); + assert_eq!(ColorChoice::Never, args.color); + + let args = + parse_low_raw(["--color", "always", "--color", "never"]).unwrap(); + assert_eq!(ColorChoice::Never, args.color); + + let args = + parse_low_raw(["--color", "never", "--color", "always"]).unwrap(); + assert_eq!(ColorChoice::Always, args.color); + + let result = parse_low_raw(["--color", "foofoo"]); + assert!(result.is_err(), "{result:?}"); + + let result = parse_low_raw(["--color", "Always"]); + assert!(result.is_err(), "{result:?}"); +} + +/// --colors +#[derive(Debug)] +struct Colors; + +impl Flag for Colors { + fn is_switch(&self) -> bool { + false + } + fn name_long(&self) -> &'static str { + "colors" + } + fn doc_variable(&self) -> Option<&'static str> { + Some("COLOR_SPEC") + } + fn doc_category(&self) -> Category { + Category::Output + } + fn doc_short(&self) -> &'static str { + "Configure color settings and styles." + } + fn doc_long(&self) -> &'static str { + r#" +This flag specifies color settings for use in the output. This flag may be +provided multiple times. Settings are applied iteratively. Pre-existing color +labels are limited to one of eight choices: \fBred\fP, \fBblue\fP, \fBgreen\fP, +\fBcyan\fP, \fBmagenta\fP, \fByellow\fP, \fBwhite\fP and \fBblack\fP. Styles +are limited to \fBnobold\fP, \fBbold\fP, \fBnointense\fP, \fBintense\fP, +\fBnounderline\fP, \fBunderline\fP, \fBnoitalic\fP or \fBitalic\fP. +.sp +The format of the flag is +\fB{\fP\fItype\fP\fB}:{\fP\fIattribute\fP\fB}:{\fP\fIvalue\fP\fB}\fP. +\fItype\fP should be one of \fBpath\fP, \fBline\fP, \fBcolumn\fP, +\fBhighlight\fP or \fBmatch\fP. \fIattribute\fP can be \fBfg\fP, \fBbg\fP or +\fBstyle\fP. \fIvalue\fP is either a color (for \fBfg\fP and \fBbg\fP) or a +text style. A special format, \fB{\fP\fItype\fP\fB}:none\fP, will clear all +color settings for \fItype\fP. +.sp +For example, the following command will change the match color to magenta and +the background color for line numbers to yellow: +.sp +.EX + rg \-\-colors 'match:fg:magenta' \-\-colors 'line:bg:yellow' +.EE +.sp +Another example, the following command will "highlight" the non-matching text +in matching lines: +.sp +.EX + rg \-\-colors 'highlight:bg:yellow' \-\-colors 'highlight:fg:black' +.EE +.sp +The "highlight" color type is particularly useful for contrasting matching +lines with surrounding context printed by the \flag{before-context}, +\flag{after-context}, \flag{context} or \flag{passthru} flags. +.sp +Extended colors can be used for \fIvalue\fP when the tty supports ANSI color +sequences. These are specified as either \fIx\fP (256-color) or +.IB x , x , x +(24-bit truecolor) where \fIx\fP is a number between \fB0\fP and \fB255\fP +inclusive. \fIx\fP may be given as a normal decimal number or a hexadecimal +number, which is prefixed by \fB0x\fP. +.sp +For example, the following command will change the match background color to +that represented by the rgb value (0,128,255): +.sp +.EX + rg \-\-colors 'match:bg:0,128,255' +.EE +.sp +or, equivalently, +.sp +.EX + rg \-\-colors 'match:bg:0x0,0x80,0xFF' +.EE +.sp +Note that the \fBintense\fP and \fBnointense\fP styles will have no effect when +used alongside these extended color codes. +"# + } + + fn update(&self, v: FlagValue, args: &mut LowArgs) -> anyhow::Result<()> { + let v = v.unwrap_value(); + let v = convert::str(&v)?; + args.colors.push(v.parse()?); + Ok(()) + } +} + +#[cfg(test)] +#[test] +fn test_colors() { + let args = parse_low_raw(None::<&str>).unwrap(); + assert!(args.colors.is_empty()); + + let args = parse_low_raw(["--colors", "match:fg:magenta"]).unwrap(); + assert_eq!(args.colors, vec!["match:fg:magenta".parse().unwrap()]); + + let args = parse_low_raw([ + "--colors", + "match:fg:magenta", + "--colors", + "line:bg:yellow", + ]) + .unwrap(); + assert_eq!( + args.colors, + vec![ + "match:fg:magenta".parse().unwrap(), + "line:bg:yellow".parse().unwrap() + ] + ); + + let args = parse_low_raw(["--colors", "highlight:bg:240"]).unwrap(); + assert_eq!(args.colors, vec!["highlight:bg:240".parse().unwrap()]); + + let args = parse_low_raw([ + "--colors", + "match:fg:magenta", + "--colors", + "highlight:bg:blue", + ]) + .unwrap(); + assert_eq!( + args.colors, + vec![ + "match:fg:magenta".parse().unwrap(), + "highlight:bg:blue".parse().unwrap() + ] + ); +} + +/// --column +#[derive(Debug)] +struct Column; + +impl Flag for Column { + fn is_switch(&self) -> bool { + true + } + fn name_long(&self) -> &'static str { + "column" + } + fn name_negated(&self) -> Option<&'static str> { + Some("no-column") + } + fn doc_category(&self) -> Category { + Category::Output + } + fn doc_short(&self) -> &'static str { + "Show column numbers." + } + fn doc_long(&self) -> &'static str { + r" +Show column numbers (1-based). This only shows the column numbers for the first +match on each line. This does not try to account for Unicode. One byte is equal +to one column. This implies \flag{line-number}. +.sp +When \flag{only-matching} is used, then the column numbers written correspond +to the start of each match. +" + } + + fn update(&self, v: FlagValue, args: &mut LowArgs) -> anyhow::Result<()> { + args.column = Some(v.unwrap_switch()); + Ok(()) + } +} + +#[cfg(test)] +#[test] +fn test_column() { + let args = parse_low_raw(None::<&str>).unwrap(); + assert_eq!(None, args.column); + + let args = parse_low_raw(["--column"]).unwrap(); + assert_eq!(Some(true), args.column); + + let args = parse_low_raw(["--column", "--no-column"]).unwrap(); + assert_eq!(Some(false), args.column); + + let args = parse_low_raw(["--no-column", "--column"]).unwrap(); + assert_eq!(Some(true), args.column); +} + +/// -C/--context +#[derive(Debug)] +struct Context; + +impl Flag for Context { + fn is_switch(&self) -> bool { + false + } + fn name_short(&self) -> Option { + Some(b'C') + } + fn name_long(&self) -> &'static str { + "context" + } + fn doc_variable(&self) -> Option<&'static str> { + Some("NUM") + } + fn doc_category(&self) -> Category { + Category::Output + } + fn doc_short(&self) -> &'static str { + r"Show NUM lines before and after each match." + } + fn doc_long(&self) -> &'static str { + r" +Show \fINUM\fP lines before and after each match. This is equivalent to +providing both the \flag{before-context} and \flag{after-context} flags with +the same value. +.sp +This overrides the \flag{passthru} flag. The \flag{after-context} and +\flag{before-context} flags both partially override this flag, regardless of +the order. For example, \fB\-A2 \-C1\fP is equivalent to \fB\-A2 \-B1\fP. +" + } + + fn update(&self, v: FlagValue, args: &mut LowArgs) -> anyhow::Result<()> { + args.context.set_both(convert::usize(&v.unwrap_value())?); + Ok(()) + } +} + +#[cfg(test)] +#[test] +fn test_context() { + let mkctx = |lines| { + let mut mode = ContextMode::default(); + mode.set_both(lines); + mode + }; + + let args = parse_low_raw(None::<&str>).unwrap(); + assert_eq!(ContextMode::default(), args.context); + + let args = parse_low_raw(["--context", "5"]).unwrap(); + assert_eq!(mkctx(5), args.context); + + let args = parse_low_raw(["--context=5"]).unwrap(); + assert_eq!(mkctx(5), args.context); + + let args = parse_low_raw(["-C", "5"]).unwrap(); + assert_eq!(mkctx(5), args.context); + + let args = parse_low_raw(["-C5"]).unwrap(); + assert_eq!(mkctx(5), args.context); + + let args = parse_low_raw(["-C5", "-C10"]).unwrap(); + assert_eq!(mkctx(10), args.context); + + let args = parse_low_raw(["-C5", "-C0"]).unwrap(); + assert_eq!(mkctx(0), args.context); + + let args = parse_low_raw(["-C5", "--passthru"]).unwrap(); + assert_eq!(ContextMode::Passthru, args.context); + + let args = parse_low_raw(["--passthru", "-C5"]).unwrap(); + assert_eq!(mkctx(5), args.context); + + let n = usize::MAX.to_string(); + let args = parse_low_raw(["--context", n.as_str()]).unwrap(); + assert_eq!(mkctx(usize::MAX), args.context); + + #[cfg(target_pointer_width = "64")] + { + let n = (u128::from(u64::MAX) + 1).to_string(); + let result = parse_low_raw(["--context", n.as_str()]); + assert!(result.is_err(), "{result:?}"); + } + + // Test the interaction between -A/-B and -C. Basically, -A/-B always + // partially overrides -C, regardless of where they appear relative to + // each other. This behavior is also how GNU grep works, and it also makes + // logical sense to me: -A/-B are the more specific flags. + let args = parse_low_raw(["-A1", "-C5"]).unwrap(); + let mut mode = ContextMode::default(); + mode.set_after(1); + mode.set_both(5); + assert_eq!(mode, args.context); + assert_eq!((5, 1), args.context.get_limited()); + + let args = parse_low_raw(["-B1", "-C5"]).unwrap(); + let mut mode = ContextMode::default(); + mode.set_before(1); + mode.set_both(5); + assert_eq!(mode, args.context); + assert_eq!((1, 5), args.context.get_limited()); + + let args = parse_low_raw(["-A1", "-B2", "-C5"]).unwrap(); + let mut mode = ContextMode::default(); + mode.set_before(2); + mode.set_after(1); + mode.set_both(5); + assert_eq!(mode, args.context); + assert_eq!((2, 1), args.context.get_limited()); + + // These next three are like the ones above, but with -C before -A/-B. This + // tests that -A and -B only partially override -C. That is, -C1 -A2 is + // equivalent to -B1 -A2. + let args = parse_low_raw(["-C5", "-A1"]).unwrap(); + let mut mode = ContextMode::default(); + mode.set_after(1); + mode.set_both(5); + assert_eq!(mode, args.context); + assert_eq!((5, 1), args.context.get_limited()); + + let args = parse_low_raw(["-C5", "-B1"]).unwrap(); + let mut mode = ContextMode::default(); + mode.set_before(1); + mode.set_both(5); + assert_eq!(mode, args.context); + assert_eq!((1, 5), args.context.get_limited()); + + let args = parse_low_raw(["-C5", "-A1", "-B2"]).unwrap(); + let mut mode = ContextMode::default(); + mode.set_before(2); + mode.set_after(1); + mode.set_both(5); + assert_eq!(mode, args.context); + assert_eq!((2, 1), args.context.get_limited()); +} + +/// --context-separator +#[derive(Debug)] +struct ContextSeparator; + +impl Flag for ContextSeparator { + fn is_switch(&self) -> bool { + false + } + fn name_long(&self) -> &'static str { + "context-separator" + } + fn name_negated(&self) -> Option<&'static str> { + Some("no-context-separator") + } + fn doc_variable(&self) -> Option<&'static str> { + Some("SEPARATOR") + } + fn doc_category(&self) -> Category { + Category::Output + } + fn doc_short(&self) -> &'static str { + r"Set the separator for contextual chunks." + } + fn doc_long(&self) -> &'static str { + r" +The string used to separate non-contiguous context lines in the output. This is +only used when one of the context flags is used (that is, \flag{after-context}, +\flag{before-context} or \flag{context}). Escape sequences like \fB\\x7F\fP or +\fB\\t\fP may be used. The default value is \fB\-\-\fP. +.sp +When the context separator is set to an empty string, then a line break +is still inserted. To completely disable context separators, use the +\flag-negate{context-separator} flag. +" + } + + fn update(&self, v: FlagValue, args: &mut LowArgs) -> anyhow::Result<()> { + use crate::flags::lowargs::ContextSeparator as Separator; + + args.context_separator = match v { + FlagValue::Switch(true) => { + unreachable!("flag can only be disabled") + } + FlagValue::Switch(false) => Separator::disabled(), + FlagValue::Value(v) => Separator::new(&v)?, + }; + Ok(()) + } +} + +#[cfg(test)] +#[test] +fn test_context_separator() { + use bstr::BString; + + use crate::flags::lowargs::ContextSeparator as Separator; + + let getbytes = |ctxsep: Separator| ctxsep.into_bytes().map(BString::from); + + let args = parse_low_raw(None::<&str>).unwrap(); + assert_eq!(Some(BString::from("--")), getbytes(args.context_separator)); + + let args = parse_low_raw(["--context-separator", "XYZ"]).unwrap(); + assert_eq!(Some(BString::from("XYZ")), getbytes(args.context_separator)); + + let args = parse_low_raw(["--no-context-separator"]).unwrap(); + assert_eq!(None, getbytes(args.context_separator)); + + let args = parse_low_raw([ + "--context-separator", + "XYZ", + "--no-context-separator", + ]) + .unwrap(); + assert_eq!(None, getbytes(args.context_separator)); + + let args = parse_low_raw([ + "--no-context-separator", + "--context-separator", + "XYZ", + ]) + .unwrap(); + assert_eq!(Some(BString::from("XYZ")), getbytes(args.context_separator)); + + // This checks that invalid UTF-8 can be used. This case isn't too tricky + // to handle, because it passes the invalid UTF-8 as an escape sequence + // that is itself valid UTF-8. It doesn't become invalid UTF-8 until after + // the argument is parsed and then unescaped. + let args = parse_low_raw(["--context-separator", r"\xFF"]).unwrap(); + assert_eq!(Some(BString::from(b"\xFF")), getbytes(args.context_separator)); + + // In this case, we specifically try to pass an invalid UTF-8 argument to + // the flag. In theory we might be able to support this, but because we do + // unescaping and because unescaping wants valid UTF-8, we do a UTF-8 check + // on the value. Since we pass invalid UTF-8, it fails. This demonstrates + // that the only way to use an invalid UTF-8 separator is by specifying an + // escape sequence that is itself valid UTF-8. + #[cfg(unix)] + { + use std::{ffi::OsStr, os::unix::ffi::OsStrExt}; + + let result = parse_low_raw([ + OsStr::from_bytes(b"--context-separator"), + OsStr::from_bytes(&[0xFF]), + ]); + assert!(result.is_err(), "{result:?}"); + } +} + +/// -c/--count +#[derive(Debug)] +struct Count; + +impl Flag for Count { + fn is_switch(&self) -> bool { + true + } + fn name_short(&self) -> Option { + Some(b'c') + } + fn name_long(&self) -> &'static str { + "count" + } + fn doc_category(&self) -> Category { + Category::OutputModes + } + fn doc_short(&self) -> &'static str { + r"Show count of matching lines for each file." + } + fn doc_long(&self) -> &'static str { + r" +This flag suppresses normal output and shows the number of lines that match +the given patterns for each file searched. Each file containing a match has +its path and count printed on each line. Note that unless \flag{multiline} is +enabled and the pattern(s) given can match over multiple lines, this reports +the number of lines that match and not the total number of matches. When +multiline mode is enabled and the pattern(s) given can match over multiple +lines, \flag{count} is equivalent to \flag{count-matches}. +.sp +If only one file is given to ripgrep, then only the count is printed if there +is a match. The \flag{with-filename} flag can be used to force printing the +file path in this case. If you need a count to be printed regardless of whether +there is a match, then use \flag{include-zero}. +.sp +Note that it is possible for this flag to have results inconsistent with +the output of \flag{files-with-matches}. Notably, by default, ripgrep tries +to avoid searching files with binary data. With this flag, ripgrep needs to +search the entire content of files, which may include binary data. But with +\flag{files-with-matches}, ripgrep can stop as soon as a match is observed, +which may come well before any binary data. To avoid this inconsistency without +disabling binary detection, use the \flag{binary} flag. +.sp +This overrides the \flag{count-matches} flag. Note that when \flag{count} +is combined with \flag{only-matching}, then ripgrep behaves as if +\flag{count-matches} was given. +" + } + + fn update(&self, v: FlagValue, args: &mut LowArgs) -> anyhow::Result<()> { + assert!(v.unwrap_switch(), "--count can only be enabled"); + args.mode.update(Mode::Search(SearchMode::Count)); + Ok(()) + } +} + +#[cfg(test)] +#[test] +fn test_count() { + let args = parse_low_raw(None::<&str>).unwrap(); + assert_eq!(Mode::Search(SearchMode::Standard), args.mode); + + let args = parse_low_raw(["--count"]).unwrap(); + assert_eq!(Mode::Search(SearchMode::Count), args.mode); + + let args = parse_low_raw(["-c"]).unwrap(); + assert_eq!(Mode::Search(SearchMode::Count), args.mode); + + let args = parse_low_raw(["--count-matches", "--count"]).unwrap(); + assert_eq!(Mode::Search(SearchMode::Count), args.mode); + + let args = parse_low_raw(["--count-matches", "-c"]).unwrap(); + assert_eq!(Mode::Search(SearchMode::Count), args.mode); +} + +/// --count-matches +#[derive(Debug)] +struct CountMatches; + +impl Flag for CountMatches { + fn is_switch(&self) -> bool { + true + } + fn name_long(&self) -> &'static str { + "count-matches" + } + fn doc_variable(&self) -> Option<&'static str> { + None + } + fn doc_category(&self) -> Category { + Category::OutputModes + } + fn doc_short(&self) -> &'static str { + r"Show count of every match for each file." + } + fn doc_long(&self) -> &'static str { + r" +This flag suppresses normal output and shows the number of individual matches +of the given patterns for each file searched. Each file containing matches has +its path and match count printed on each line. Note that this reports the total +number of individual matches and not the number of lines that match. +.sp +If only one file is given to ripgrep, then only the count is printed if there +is a match. The \flag{with-filename} flag can be used to force printing the +file path in this case. +.sp +This overrides the \flag{count} flag. Note that when \flag{count} is combined +with \flag{only-matching}, then ripgrep behaves as if \flag{count-matches} was +given. +" + } + + fn update(&self, v: FlagValue, args: &mut LowArgs) -> anyhow::Result<()> { + assert!(v.unwrap_switch(), "--count-matches can only be enabled"); + args.mode.update(Mode::Search(SearchMode::CountMatches)); + Ok(()) + } +} + +#[cfg(test)] +#[test] +fn test_count_matches() { + let args = parse_low_raw(None::<&str>).unwrap(); + assert_eq!(Mode::Search(SearchMode::Standard), args.mode); + + let args = parse_low_raw(["--count-matches"]).unwrap(); + assert_eq!(Mode::Search(SearchMode::CountMatches), args.mode); + + let args = parse_low_raw(["--count", "--count-matches"]).unwrap(); + assert_eq!(Mode::Search(SearchMode::CountMatches), args.mode); + + let args = parse_low_raw(["-c", "--count-matches"]).unwrap(); + assert_eq!(Mode::Search(SearchMode::CountMatches), args.mode); +} + +/// --crlf +#[derive(Debug)] +struct Crlf; + +impl Flag for Crlf { + fn is_switch(&self) -> bool { + true + } + fn name_long(&self) -> &'static str { + "crlf" + } + fn name_negated(&self) -> Option<&'static str> { + Some("no-crlf") + } + fn doc_category(&self) -> Category { + Category::Search + } + fn doc_short(&self) -> &'static str { + r"Use CRLF line terminators (nice for Windows)." + } + fn doc_long(&self) -> &'static str { + r" +When enabled, ripgrep will treat CRLF (\fB\\r\\n\fP) as a line terminator +instead of just \fB\\n\fP. +.sp +Principally, this permits the line anchor assertions \fB^\fP and \fB$\fP in +regex patterns to treat CRLF, CR or LF as line terminators instead of just LF. +Note that they will never match between a CR and a LF. CRLF is treated as one +single line terminator. +.sp +When using the default regex engine, CRLF support can also be enabled inside +the pattern with the \fBR\fP flag. For example, \fB(?R:$)\fP will match just +before either CR or LF, but never between CR and LF. +.sp +This flag overrides \flag{null-data}. +" + } + + fn update(&self, v: FlagValue, args: &mut LowArgs) -> anyhow::Result<()> { + args.crlf = v.unwrap_switch(); + if args.crlf { + args.null_data = false; + } + Ok(()) + } +} + +#[cfg(test)] +#[test] +fn test_crlf() { + let args = parse_low_raw(None::<&str>).unwrap(); + assert_eq!(false, args.crlf); + + let args = parse_low_raw(["--crlf"]).unwrap(); + assert_eq!(true, args.crlf); + assert_eq!(false, args.null_data); + + let args = parse_low_raw(["--crlf", "--null-data"]).unwrap(); + assert_eq!(false, args.crlf); + assert_eq!(true, args.null_data); + + let args = parse_low_raw(["--null-data", "--crlf"]).unwrap(); + assert_eq!(true, args.crlf); + assert_eq!(false, args.null_data); + + let args = parse_low_raw(["--null-data", "--no-crlf"]).unwrap(); + assert_eq!(false, args.crlf); + assert_eq!(true, args.null_data); + + let args = parse_low_raw(["--null-data", "--crlf", "--no-crlf"]).unwrap(); + assert_eq!(false, args.crlf); + assert_eq!(false, args.null_data); +} + +/// --debug +#[derive(Debug)] +struct Debug; + +impl Flag for Debug { + fn is_switch(&self) -> bool { + true + } + fn name_long(&self) -> &'static str { + "debug" + } + fn doc_category(&self) -> Category { + Category::Logging + } + fn doc_short(&self) -> &'static str { + r"Show debug messages." + } + fn doc_long(&self) -> &'static str { + r" +Show debug messages. Please use this when filing a bug report. +.sp +The \flag{debug} flag is generally useful for figuring out why ripgrep skipped +searching a particular file. The debug messages should mention all files +skipped and why they were skipped. +.sp +To get even more debug output, use the \flag{trace} flag, which implies +\flag{debug} along with additional trace data. +" + } + + fn update(&self, v: FlagValue, args: &mut LowArgs) -> anyhow::Result<()> { + assert!(v.unwrap_switch(), "--debug can only be enabled"); + args.logging = Some(LoggingMode::Debug); + Ok(()) + } +} + +#[cfg(test)] +#[test] +fn test_debug() { + let args = parse_low_raw(None::<&str>).unwrap(); + assert_eq!(None, args.logging); + + let args = parse_low_raw(["--debug"]).unwrap(); + assert_eq!(Some(LoggingMode::Debug), args.logging); + + let args = parse_low_raw(["--trace", "--debug"]).unwrap(); + assert_eq!(Some(LoggingMode::Debug), args.logging); +} + +/// --dfa-size-limit +#[derive(Debug)] +struct DfaSizeLimit; + +impl Flag for DfaSizeLimit { + fn is_switch(&self) -> bool { + false + } + fn name_long(&self) -> &'static str { + "dfa-size-limit" + } + fn doc_variable(&self) -> Option<&'static str> { + Some("NUM+SUFFIX?") + } + fn doc_category(&self) -> Category { + Category::Search + } + fn doc_short(&self) -> &'static str { + r"The upper size limit of the regex DFA." + } + fn doc_long(&self) -> &'static str { + r" +The upper size limit of the regex DFA. The default limit is something generous +for any single pattern or for many smallish patterns. This should only be +changed on very large regex inputs where the (slower) fallback regex engine may +otherwise be used if the limit is reached. +.sp +The input format accepts suffixes of \fBK\fP, \fBM\fP or \fBG\fP which +correspond to kilobytes, megabytes and gigabytes, respectively. If no suffix is +provided the input is treated as bytes. +" + } + + fn update(&self, v: FlagValue, args: &mut LowArgs) -> anyhow::Result<()> { + let v = v.unwrap_value(); + args.dfa_size_limit = Some(convert::human_readable_usize(&v)?); + Ok(()) + } +} + +#[cfg(test)] +#[test] +fn test_dfa_size_limit() { + let args = parse_low_raw(None::<&str>).unwrap(); + assert_eq!(None, args.dfa_size_limit); + + #[cfg(target_pointer_width = "64")] + { + let args = parse_low_raw(["--dfa-size-limit", "9G"]).unwrap(); + assert_eq!(Some(9 * (1 << 30)), args.dfa_size_limit); + + let args = parse_low_raw(["--dfa-size-limit=9G"]).unwrap(); + assert_eq!(Some(9 * (1 << 30)), args.dfa_size_limit); + + let args = + parse_low_raw(["--dfa-size-limit=9G", "--dfa-size-limit=0"]) + .unwrap(); + assert_eq!(Some(0), args.dfa_size_limit); + } + + let args = parse_low_raw(["--dfa-size-limit=0K"]).unwrap(); + assert_eq!(Some(0), args.dfa_size_limit); + + let args = parse_low_raw(["--dfa-size-limit=0M"]).unwrap(); + assert_eq!(Some(0), args.dfa_size_limit); + + let args = parse_low_raw(["--dfa-size-limit=0G"]).unwrap(); + assert_eq!(Some(0), args.dfa_size_limit); + + let result = parse_low_raw(["--dfa-size-limit", "9999999999999999999999"]); + assert!(result.is_err(), "{result:?}"); + + let result = parse_low_raw(["--dfa-size-limit", "9999999999999999G"]); + assert!(result.is_err(), "{result:?}"); +} + +/// -E/--encoding +#[derive(Debug)] +struct Encoding; + +impl Flag for Encoding { + fn is_switch(&self) -> bool { + false + } + fn name_short(&self) -> Option { + Some(b'E') + } + fn name_long(&self) -> &'static str { + "encoding" + } + fn name_negated(&self) -> Option<&'static str> { + Some("no-encoding") + } + fn doc_variable(&self) -> Option<&'static str> { + Some("ENCODING") + } + fn doc_category(&self) -> Category { + Category::Search + } + fn doc_short(&self) -> &'static str { + r"Specify the text encoding of files to search." + } + fn doc_long(&self) -> &'static str { + r" +Specify the text encoding that ripgrep will use on all files searched. The +default value is \fBauto\fP, which will cause ripgrep to do a best effort +automatic detection of encoding on a per-file basis. Automatic detection in +this case only applies to files that begin with a UTF-8 or UTF-16 byte-order +mark (BOM). No other automatic detection is performed. One can also specify +\fBnone\fP which will then completely disable BOM sniffing and always result +in searching the raw bytes, including a BOM if it's present, regardless of its +encoding. +.sp +Other supported values can be found in the list of labels here: +\fIhttps://encoding.spec.whatwg.org/#concept-encoding-get\fP. +.sp +For more details on encoding and how ripgrep deals with it, see \fBGUIDE.md\fP. +.sp +The encoding detection that ripgrep uses can be reverted to its automatic mode +via the \flag-negate{encoding} flag. +" + } + fn completion_type(&self) -> CompletionType { + CompletionType::Encoding + } + + fn update(&self, v: FlagValue, args: &mut LowArgs) -> anyhow::Result<()> { + let value = match v { + FlagValue::Value(v) => v, + FlagValue::Switch(true) => { + unreachable!("--encoding must accept a value") + } + FlagValue::Switch(false) => { + args.encoding = EncodingMode::Auto; + return Ok(()); + } + }; + let label = convert::str(&value)?; + args.encoding = match label { + "auto" => EncodingMode::Auto, + "none" => EncodingMode::Disabled, + _ => EncodingMode::Some(grep::searcher::Encoding::new(label)?), + }; + Ok(()) + } +} + +#[cfg(test)] +#[test] +fn test_encoding() { + let args = parse_low_raw(None::<&str>).unwrap(); + assert_eq!(EncodingMode::Auto, args.encoding); + + let args = parse_low_raw(["--encoding", "auto"]).unwrap(); + assert_eq!(EncodingMode::Auto, args.encoding); + + let args = parse_low_raw(["--encoding", "none"]).unwrap(); + assert_eq!(EncodingMode::Disabled, args.encoding); + + let args = parse_low_raw(["--encoding=none"]).unwrap(); + assert_eq!(EncodingMode::Disabled, args.encoding); + + let args = parse_low_raw(["-E", "none"]).unwrap(); + assert_eq!(EncodingMode::Disabled, args.encoding); + + let args = parse_low_raw(["-Enone"]).unwrap(); + assert_eq!(EncodingMode::Disabled, args.encoding); + + let args = parse_low_raw(["-E", "none", "--no-encoding"]).unwrap(); + assert_eq!(EncodingMode::Auto, args.encoding); + + let args = parse_low_raw(["--no-encoding", "-E", "none"]).unwrap(); + assert_eq!(EncodingMode::Disabled, args.encoding); + + let args = parse_low_raw(["-E", "utf-16"]).unwrap(); + let enc = grep::searcher::Encoding::new("utf-16").unwrap(); + assert_eq!(EncodingMode::Some(enc), args.encoding); + + let args = parse_low_raw(["-E", "utf-16", "--no-encoding"]).unwrap(); + assert_eq!(EncodingMode::Auto, args.encoding); + + let result = parse_low_raw(["-E", "foo"]); + assert!(result.is_err(), "{result:?}"); +} + +/// --engine +#[derive(Debug)] +struct Engine; + +impl Flag for Engine { + fn is_switch(&self) -> bool { + false + } + fn name_long(&self) -> &'static str { + "engine" + } + fn doc_variable(&self) -> Option<&'static str> { + Some("ENGINE") + } + fn doc_category(&self) -> Category { + Category::Search + } + fn doc_short(&self) -> &'static str { + r"Specify which regex engine to use." + } + fn doc_long(&self) -> &'static str { + r" +Specify which regular expression engine to use. When you choose a regex engine, +it applies that choice for every regex provided to ripgrep (e.g., via multiple +\flag{regexp} or \flag{file} flags). +.sp +Accepted values are \fBdefault\fP, \fBpcre2\fP, or \fBauto\fP. +.sp +The default value is \fBdefault\fP, which is usually the fastest and should be +good for most use cases. The \fBpcre2\fP engine is generally useful when you +want to use features such as look-around or backreferences. \fBauto\fP will +dynamically choose between supported regex engines depending on the features +used in a pattern on a best effort basis. +.sp +Note that the \fBpcre2\fP engine is an optional ripgrep feature. If PCRE2 +wasn't included in your build of ripgrep, then using this flag will result in +ripgrep printing an error message and exiting. +.sp +This overrides previous uses of the \flag{pcre2} and \flag{auto-hybrid-regex} +flags. +" + } + fn doc_choices(&self) -> &'static [&'static str] { + &["default", "pcre2", "auto"] + } + + fn update(&self, v: FlagValue, args: &mut LowArgs) -> anyhow::Result<()> { + let v = v.unwrap_value(); + let string = convert::str(&v)?; + args.engine = match string { + "default" => EngineChoice::Default, + "pcre2" => EngineChoice::PCRE2, + "auto" => EngineChoice::Auto, + _ => anyhow::bail!("unrecognized regex engine '{string}'"), + }; + Ok(()) + } +} + +#[cfg(test)] +#[test] +fn test_engine() { + let args = parse_low_raw(None::<&str>).unwrap(); + assert_eq!(EngineChoice::Default, args.engine); + + let args = parse_low_raw(["--engine", "pcre2"]).unwrap(); + assert_eq!(EngineChoice::PCRE2, args.engine); + + let args = parse_low_raw(["--engine=pcre2"]).unwrap(); + assert_eq!(EngineChoice::PCRE2, args.engine); + + let args = + parse_low_raw(["--auto-hybrid-regex", "--engine=pcre2"]).unwrap(); + assert_eq!(EngineChoice::PCRE2, args.engine); + + let args = + parse_low_raw(["--engine=pcre2", "--auto-hybrid-regex"]).unwrap(); + assert_eq!(EngineChoice::Auto, args.engine); + + let args = + parse_low_raw(["--auto-hybrid-regex", "--engine=auto"]).unwrap(); + assert_eq!(EngineChoice::Auto, args.engine); + + let args = + parse_low_raw(["--auto-hybrid-regex", "--engine=default"]).unwrap(); + assert_eq!(EngineChoice::Default, args.engine); + + let args = + parse_low_raw(["--engine=pcre2", "--no-auto-hybrid-regex"]).unwrap(); + assert_eq!(EngineChoice::Default, args.engine); +} + +/// --field-context-separator +#[derive(Debug)] +struct FieldContextSeparator; + +impl Flag for FieldContextSeparator { + fn is_switch(&self) -> bool { + false + } + fn name_long(&self) -> &'static str { + "field-context-separator" + } + fn doc_variable(&self) -> Option<&'static str> { + Some("SEPARATOR") + } + fn doc_category(&self) -> Category { + Category::Output + } + fn doc_short(&self) -> &'static str { + r"Set the field context separator." + } + fn doc_long(&self) -> &'static str { + r" +Set the field context separator. This separator is only used when printing +contextual lines. It is used to delimit file paths, line numbers, columns and +the contextual line itself. The separator may be any number of bytes, including +zero. Escape sequences like \fB\\x7F\fP or \fB\\t\fP may be used. +.sp +The \fB-\fP character is the default value. +" + } + + fn update(&self, v: FlagValue, args: &mut LowArgs) -> anyhow::Result<()> { + use crate::flags::lowargs::FieldContextSeparator as Separator; + + args.field_context_separator = Separator::new(&v.unwrap_value())?; + Ok(()) + } +} + +#[cfg(test)] +#[test] +fn test_field_context_separator() { + use bstr::BString; + + let args = parse_low_raw(None::<&str>).unwrap(); + assert_eq!(BString::from("-"), args.field_context_separator.into_bytes()); + + let args = parse_low_raw(["--field-context-separator", "XYZ"]).unwrap(); + assert_eq!( + BString::from("XYZ"), + args.field_context_separator.into_bytes() + ); + + let args = parse_low_raw(["--field-context-separator=XYZ"]).unwrap(); + assert_eq!( + BString::from("XYZ"), + args.field_context_separator.into_bytes() + ); + + let args = parse_low_raw([ + "--field-context-separator", + "XYZ", + "--field-context-separator", + "ABC", + ]) + .unwrap(); + assert_eq!( + BString::from("ABC"), + args.field_context_separator.into_bytes() + ); + + let args = parse_low_raw(["--field-context-separator", r"\t"]).unwrap(); + assert_eq!(BString::from("\t"), args.field_context_separator.into_bytes()); + + let args = parse_low_raw(["--field-context-separator", r"\x00"]).unwrap(); + assert_eq!( + BString::from("\x00"), + args.field_context_separator.into_bytes() + ); + + // This checks that invalid UTF-8 can be used. This case isn't too tricky + // to handle, because it passes the invalid UTF-8 as an escape sequence + // that is itself valid UTF-8. It doesn't become invalid UTF-8 until after + // the argument is parsed and then unescaped. + let args = parse_low_raw(["--field-context-separator", r"\xFF"]).unwrap(); + assert_eq!( + BString::from(b"\xFF"), + args.field_context_separator.into_bytes() + ); + + // In this case, we specifically try to pass an invalid UTF-8 argument to + // the flag. In theory we might be able to support this, but because we do + // unescaping and because unescaping wants valid UTF-8, we do a UTF-8 check + // on the value. Since we pass invalid UTF-8, it fails. This demonstrates + // that the only way to use an invalid UTF-8 separator is by specifying an + // escape sequence that is itself valid UTF-8. + #[cfg(unix)] + { + use std::{ffi::OsStr, os::unix::ffi::OsStrExt}; + + let result = parse_low_raw([ + OsStr::from_bytes(b"--field-context-separator"), + OsStr::from_bytes(&[0xFF]), + ]); + assert!(result.is_err(), "{result:?}"); + } +} + +/// --field-match-separator +#[derive(Debug)] +struct FieldMatchSeparator; + +impl Flag for FieldMatchSeparator { + fn is_switch(&self) -> bool { + false + } + fn name_long(&self) -> &'static str { + "field-match-separator" + } + fn doc_variable(&self) -> Option<&'static str> { + Some("SEPARATOR") + } + fn doc_category(&self) -> Category { + Category::Output + } + fn doc_short(&self) -> &'static str { + r"Set the field match separator." + } + fn doc_long(&self) -> &'static str { + r" +Set the field match separator. This separator is only used when printing +matching lines. It is used to delimit file paths, line numbers, columns and the +matching line itself. The separator may be any number of bytes, including zero. +Escape sequences like \fB\\x7F\fP or \fB\\t\fP may be used. +.sp +The \fB:\fP character is the default value. +" + } + + fn update(&self, v: FlagValue, args: &mut LowArgs) -> anyhow::Result<()> { + use crate::flags::lowargs::FieldMatchSeparator as Separator; + + args.field_match_separator = Separator::new(&v.unwrap_value())?; + Ok(()) + } +} + +#[cfg(test)] +#[test] +fn test_field_match_separator() { + use bstr::BString; + + let args = parse_low_raw(None::<&str>).unwrap(); + assert_eq!(BString::from(":"), args.field_match_separator.into_bytes()); + + let args = parse_low_raw(["--field-match-separator", "XYZ"]).unwrap(); + assert_eq!(BString::from("XYZ"), args.field_match_separator.into_bytes()); + + let args = parse_low_raw(["--field-match-separator=XYZ"]).unwrap(); + assert_eq!(BString::from("XYZ"), args.field_match_separator.into_bytes()); + + let args = parse_low_raw([ + "--field-match-separator", + "XYZ", + "--field-match-separator", + "ABC", + ]) + .unwrap(); + assert_eq!(BString::from("ABC"), args.field_match_separator.into_bytes()); + + let args = parse_low_raw(["--field-match-separator", r"\t"]).unwrap(); + assert_eq!(BString::from("\t"), args.field_match_separator.into_bytes()); + + let args = parse_low_raw(["--field-match-separator", r"\x00"]).unwrap(); + assert_eq!(BString::from("\x00"), args.field_match_separator.into_bytes()); + + // This checks that invalid UTF-8 can be used. This case isn't too tricky + // to handle, because it passes the invalid UTF-8 as an escape sequence + // that is itself valid UTF-8. It doesn't become invalid UTF-8 until after + // the argument is parsed and then unescaped. + let args = parse_low_raw(["--field-match-separator", r"\xFF"]).unwrap(); + assert_eq!( + BString::from(b"\xFF"), + args.field_match_separator.into_bytes() + ); + + // In this case, we specifically try to pass an invalid UTF-8 argument to + // the flag. In theory we might be able to support this, but because we do + // unescaping and because unescaping wants valid UTF-8, we do a UTF-8 check + // on the value. Since we pass invalid UTF-8, it fails. This demonstrates + // that the only way to use an invalid UTF-8 separator is by specifying an + // escape sequence that is itself valid UTF-8. + #[cfg(unix)] + { + use std::{ffi::OsStr, os::unix::ffi::OsStrExt}; + + let result = parse_low_raw([ + OsStr::from_bytes(b"--field-match-separator"), + OsStr::from_bytes(&[0xFF]), + ]); + assert!(result.is_err(), "{result:?}"); + } +} + +/// -f/--file +#[derive(Debug)] +struct File; + +impl Flag for File { + fn is_switch(&self) -> bool { + false + } + fn name_short(&self) -> Option { + Some(b'f') + } + fn name_long(&self) -> &'static str { + "file" + } + fn doc_variable(&self) -> Option<&'static str> { + Some("PATTERNFILE") + } + fn doc_category(&self) -> Category { + Category::Input + } + fn doc_short(&self) -> &'static str { + r"Search for patterns from the given file." + } + fn doc_long(&self) -> &'static str { + r" +Search for patterns from the given file, with one pattern per line. When this +flag is used multiple times or in combination with the \flag{regexp} flag, then +all patterns provided are searched. Empty pattern lines will match all input +lines, and the newline is not counted as part of the pattern. +.sp +A line is printed if and only if it matches at least one of the patterns. +.sp +When \fIPATTERNFILE\fP is \fB-\fP, then \fBstdin\fP will be read for the +patterns. +.sp +When \flag{file} or \flag{regexp} is used, then ripgrep treats all positional +arguments as files or directories to search. +" + } + fn completion_type(&self) -> CompletionType { + CompletionType::Filename + } + + fn update(&self, v: FlagValue, args: &mut LowArgs) -> anyhow::Result<()> { + let path = PathBuf::from(v.unwrap_value()); + args.patterns.push(PatternSource::File(path)); + Ok(()) + } +} + +#[cfg(test)] +#[test] +fn test_file() { + let args = parse_low_raw(None::<&str>).unwrap(); + assert_eq!(Vec::::new(), args.patterns); + + let args = parse_low_raw(["--file", "foo"]).unwrap(); + assert_eq!(vec![PatternSource::File(PathBuf::from("foo"))], args.patterns); + + let args = parse_low_raw(["--file=foo"]).unwrap(); + assert_eq!(vec![PatternSource::File(PathBuf::from("foo"))], args.patterns); + + let args = parse_low_raw(["-f", "foo"]).unwrap(); + assert_eq!(vec![PatternSource::File(PathBuf::from("foo"))], args.patterns); + + let args = parse_low_raw(["-ffoo"]).unwrap(); + assert_eq!(vec![PatternSource::File(PathBuf::from("foo"))], args.patterns); + + let args = parse_low_raw(["--file", "-foo"]).unwrap(); + assert_eq!( + vec![PatternSource::File(PathBuf::from("-foo"))], + args.patterns + ); + + let args = parse_low_raw(["--file=-foo"]).unwrap(); + assert_eq!( + vec![PatternSource::File(PathBuf::from("-foo"))], + args.patterns + ); + + let args = parse_low_raw(["-f", "-foo"]).unwrap(); + assert_eq!( + vec![PatternSource::File(PathBuf::from("-foo"))], + args.patterns + ); + + let args = parse_low_raw(["-f-foo"]).unwrap(); + assert_eq!( + vec![PatternSource::File(PathBuf::from("-foo"))], + args.patterns + ); + + let args = parse_low_raw(["--file=foo", "--file", "bar"]).unwrap(); + assert_eq!( + vec![ + PatternSource::File(PathBuf::from("foo")), + PatternSource::File(PathBuf::from("bar")) + ], + args.patterns + ); + + // We permit path arguments to be invalid UTF-8. So test that. Some of + // these cases are tricky and depend on lexopt doing the right thing. + // + // We probably should add tests for this handling on Windows too, but paths + // that are invalid UTF-16 appear incredibly rare in the Windows world. + #[cfg(unix)] + { + use std::{ + ffi::{OsStr, OsString}, + os::unix::ffi::{OsStrExt, OsStringExt}, + }; + + let bytes = &[b'A', 0xFF, b'Z'][..]; + let path = PathBuf::from(OsString::from_vec(bytes.to_vec())); + + let args = parse_low_raw([ + OsStr::from_bytes(b"--file"), + OsStr::from_bytes(bytes), + ]) + .unwrap(); + assert_eq!(vec![PatternSource::File(path.clone())], args.patterns); + + let args = parse_low_raw([ + OsStr::from_bytes(b"-f"), + OsStr::from_bytes(bytes), + ]) + .unwrap(); + assert_eq!(vec![PatternSource::File(path.clone())], args.patterns); + + let mut bytes = b"--file=A".to_vec(); + bytes.push(0xFF); + bytes.push(b'Z'); + let args = parse_low_raw([OsStr::from_bytes(&bytes)]).unwrap(); + assert_eq!(vec![PatternSource::File(path.clone())], args.patterns); + + let mut bytes = b"-fA".to_vec(); + bytes.push(0xFF); + bytes.push(b'Z'); + let args = parse_low_raw([OsStr::from_bytes(&bytes)]).unwrap(); + assert_eq!(vec![PatternSource::File(path.clone())], args.patterns); + } +} + +/// --files +#[derive(Debug)] +struct Files; + +impl Flag for Files { + fn is_switch(&self) -> bool { + true + } + fn name_long(&self) -> &'static str { + "files" + } + fn doc_category(&self) -> Category { + Category::OtherBehaviors + } + fn doc_short(&self) -> &'static str { + r"Print each file that would be searched." + } + fn doc_long(&self) -> &'static str { + r" +Print each file that would be searched without actually performing the search. +This is useful to determine whether a particular file is being searched or not. +.sp +This overrides \flag{type-list}. +" + } + + fn update(&self, v: FlagValue, args: &mut LowArgs) -> anyhow::Result<()> { + assert!(v.unwrap_switch()); + args.mode.update(Mode::Files); + Ok(()) + } +} + +#[cfg(test)] +#[test] +fn test_files() { + let args = parse_low_raw(None::<&str>).unwrap(); + assert_eq!(Mode::Search(SearchMode::Standard), args.mode); + + let args = parse_low_raw(["--files"]).unwrap(); + assert_eq!(Mode::Files, args.mode); +} + +/// -l/--files-with-matches +#[derive(Debug)] +struct FilesWithMatches; + +impl Flag for FilesWithMatches { + fn is_switch(&self) -> bool { + true + } + fn name_short(&self) -> Option { + Some(b'l') + } + fn name_long(&self) -> &'static str { + "files-with-matches" + } + fn doc_category(&self) -> Category { + Category::OutputModes + } + fn doc_short(&self) -> &'static str { + r"Print the paths with at least one match." + } + fn doc_long(&self) -> &'static str { + r" +Print only the paths with at least one match and suppress match contents. +.sp +Note that it is possible for this flag to have results inconsistent with the +output of \flag{count}. Notably, by default, ripgrep tries to avoid searching +files with binary data. With this flag, ripgrep might stop searching before +the binary data is observed. But with \flag{count}, ripgrep has to search the +entire contents to determine the match count, which means it might see binary +data that causes it to skip searching that file. To avoid this inconsistency +without disabling binary detection, use the \flag{binary} flag. +.sp +This overrides \flag{files-without-match}. +" + } + + fn update(&self, v: FlagValue, args: &mut LowArgs) -> anyhow::Result<()> { + assert!(v.unwrap_switch(), "--files-with-matches can only be enabled"); + args.mode.update(Mode::Search(SearchMode::FilesWithMatches)); + Ok(()) + } +} + +#[cfg(test)] +#[test] +fn test_files_with_matches() { + let args = parse_low_raw(None::<&str>).unwrap(); + assert_eq!(Mode::Search(SearchMode::Standard), args.mode); + + let args = parse_low_raw(["--files-with-matches"]).unwrap(); + assert_eq!(Mode::Search(SearchMode::FilesWithMatches), args.mode); + + let args = parse_low_raw(["-l"]).unwrap(); + assert_eq!(Mode::Search(SearchMode::FilesWithMatches), args.mode); +} + +/// -l/--files-without-match +#[derive(Debug)] +struct FilesWithoutMatch; + +impl Flag for FilesWithoutMatch { + fn is_switch(&self) -> bool { + true + } + fn name_long(&self) -> &'static str { + "files-without-match" + } + fn doc_category(&self) -> Category { + Category::OutputModes + } + fn doc_short(&self) -> &'static str { + r"Print the paths that contain zero matches." + } + fn doc_long(&self) -> &'static str { + r" +Print the paths that contain zero matches and suppress match contents. +.sp +This overrides \flag{files-with-matches}. +" + } + + fn update(&self, v: FlagValue, args: &mut LowArgs) -> anyhow::Result<()> { + assert!( + v.unwrap_switch(), + "--files-without-match can only be enabled" + ); + args.mode.update(Mode::Search(SearchMode::FilesWithoutMatch)); + Ok(()) + } +} + +#[cfg(test)] +#[test] +fn test_files_without_match() { + let args = parse_low_raw(None::<&str>).unwrap(); + assert_eq!(Mode::Search(SearchMode::Standard), args.mode); + + let args = parse_low_raw(["--files-without-match"]).unwrap(); + assert_eq!(Mode::Search(SearchMode::FilesWithoutMatch), args.mode); + + let args = + parse_low_raw(["--files-with-matches", "--files-without-match"]) + .unwrap(); + assert_eq!(Mode::Search(SearchMode::FilesWithoutMatch), args.mode); + + let args = + parse_low_raw(["--files-without-match", "--files-with-matches"]) + .unwrap(); + assert_eq!(Mode::Search(SearchMode::FilesWithMatches), args.mode); +} + +/// -F/--fixed-strings +#[derive(Debug)] +struct FixedStrings; + +impl Flag for FixedStrings { + fn is_switch(&self) -> bool { + true + } + fn name_short(&self) -> Option { + Some(b'F') + } + fn name_long(&self) -> &'static str { + "fixed-strings" + } + fn name_negated(&self) -> Option<&'static str> { + Some("no-fixed-strings") + } + fn doc_category(&self) -> Category { + Category::Search + } + fn doc_short(&self) -> &'static str { + r"Treat all patterns as literals." + } + fn doc_long(&self) -> &'static str { + r" +Treat all patterns as literals instead of as regular expressions. When this +flag is used, special regular expression meta characters such as \fB.(){}*+\fP +should not need be escaped. +" + } + + fn update(&self, v: FlagValue, args: &mut LowArgs) -> anyhow::Result<()> { + args.fixed_strings = v.unwrap_switch(); + Ok(()) + } +} + +#[cfg(test)] +#[test] +fn test_fixed_strings() { + let args = parse_low_raw(None::<&str>).unwrap(); + assert_eq!(false, args.fixed_strings); + + let args = parse_low_raw(["--fixed-strings"]).unwrap(); + assert_eq!(true, args.fixed_strings); + + let args = parse_low_raw(["-F"]).unwrap(); + assert_eq!(true, args.fixed_strings); + + let args = parse_low_raw(["-F", "--no-fixed-strings"]).unwrap(); + assert_eq!(false, args.fixed_strings); + + let args = parse_low_raw(["--no-fixed-strings", "-F"]).unwrap(); + assert_eq!(true, args.fixed_strings); +} + +/// -L/--follow +#[derive(Debug)] +struct Follow; + +impl Flag for Follow { + fn is_switch(&self) -> bool { + true + } + fn name_short(&self) -> Option { + Some(b'L') + } + fn name_long(&self) -> &'static str { + "follow" + } + fn name_negated(&self) -> Option<&'static str> { + Some("no-follow") + } + fn doc_category(&self) -> Category { + Category::Filter + } + fn doc_short(&self) -> &'static str { + r"Follow symbolic links." + } + fn doc_long(&self) -> &'static str { + r" +This flag instructs ripgrep to follow symbolic links while traversing +directories. This behavior is disabled by default. Note that ripgrep will +check for symbolic link loops and report errors if it finds one. ripgrep will +also report errors for broken links. To suppress error messages, use the +\flag{no-messages} flag. +" + } + + fn update(&self, v: FlagValue, args: &mut LowArgs) -> anyhow::Result<()> { + args.follow = v.unwrap_switch(); + Ok(()) + } +} + +#[cfg(test)] +#[test] +fn test_follow() { + let args = parse_low_raw(None::<&str>).unwrap(); + assert_eq!(false, args.follow); + + let args = parse_low_raw(["--follow"]).unwrap(); + assert_eq!(true, args.follow); + + let args = parse_low_raw(["-L"]).unwrap(); + assert_eq!(true, args.follow); + + let args = parse_low_raw(["-L", "--no-follow"]).unwrap(); + assert_eq!(false, args.follow); + + let args = parse_low_raw(["--no-follow", "-L"]).unwrap(); + assert_eq!(true, args.follow); +} + +/// --generate +#[derive(Debug)] +struct Generate; + +impl Flag for Generate { + fn is_switch(&self) -> bool { + false + } + fn name_long(&self) -> &'static str { + "generate" + } + fn doc_variable(&self) -> Option<&'static str> { + Some("KIND") + } + fn doc_category(&self) -> Category { + Category::OtherBehaviors + } + fn doc_short(&self) -> &'static str { + r"Generate man pages and completion scripts." + } + fn doc_long(&self) -> &'static str { + r" +This flag instructs ripgrep to generate some special kind of output identified +by \fIKIND\fP and then quit without searching. \fIKIND\fP can be one of the +following values: +.sp +.TP 15 +\fBman\fP +Generates a manual page for ripgrep in the \fBroff\fP format. +.TP 15 +\fBcomplete\-bash\fP +Generates a completion script for the \fBbash\fP shell. +.TP 15 +\fBcomplete\-zsh\fP +Generates a completion script for the \fBzsh\fP shell. +.TP 15 +\fBcomplete\-fish\fP +Generates a completion script for the \fBfish\fP shell. +.TP 15 +\fBcomplete\-powershell\fP +Generates a completion script for PowerShell. +.PP +The output is written to \fBstdout\fP. The list above may expand over time. +" + } + fn doc_choices(&self) -> &'static [&'static str] { + &[ + "man", + "complete-bash", + "complete-zsh", + "complete-fish", + "complete-powershell", + ] + } + + fn update(&self, v: FlagValue, args: &mut LowArgs) -> anyhow::Result<()> { + let genmode = match convert::str(&v.unwrap_value())? { + "man" => GenerateMode::Man, + "complete-bash" => GenerateMode::CompleteBash, + "complete-zsh" => GenerateMode::CompleteZsh, + "complete-fish" => GenerateMode::CompleteFish, + "complete-powershell" => GenerateMode::CompletePowerShell, + unk => anyhow::bail!("choice '{unk}' is unrecognized"), + }; + args.mode.update(Mode::Generate(genmode)); + Ok(()) + } +} + +#[cfg(test)] +#[test] +fn test_generate() { + let args = parse_low_raw(None::<&str>).unwrap(); + assert_eq!(Mode::Search(SearchMode::Standard), args.mode); + + let args = parse_low_raw(["--generate", "man"]).unwrap(); + assert_eq!(Mode::Generate(GenerateMode::Man), args.mode); + + let args = parse_low_raw(["--generate", "complete-bash"]).unwrap(); + assert_eq!(Mode::Generate(GenerateMode::CompleteBash), args.mode); + + let args = parse_low_raw(["--generate", "complete-zsh"]).unwrap(); + assert_eq!(Mode::Generate(GenerateMode::CompleteZsh), args.mode); + + let args = parse_low_raw(["--generate", "complete-fish"]).unwrap(); + assert_eq!(Mode::Generate(GenerateMode::CompleteFish), args.mode); + + let args = parse_low_raw(["--generate", "complete-powershell"]).unwrap(); + assert_eq!(Mode::Generate(GenerateMode::CompletePowerShell), args.mode); + + let args = + parse_low_raw(["--generate", "complete-bash", "--generate=man"]) + .unwrap(); + assert_eq!(Mode::Generate(GenerateMode::Man), args.mode); + + let args = parse_low_raw(["--generate", "man", "-l"]).unwrap(); + assert_eq!(Mode::Search(SearchMode::FilesWithMatches), args.mode); + + // An interesting quirk of how the modes override each other that lets + // you get back to the "default" mode of searching. + let args = + parse_low_raw(["--generate", "man", "--json", "--no-json"]).unwrap(); + assert_eq!(Mode::Search(SearchMode::Standard), args.mode); +} + +/// -g/--glob +#[derive(Debug)] +struct Glob; + +impl Flag for Glob { + fn is_switch(&self) -> bool { + false + } + fn name_short(&self) -> Option { + Some(b'g') + } + fn name_long(&self) -> &'static str { + "glob" + } + fn doc_variable(&self) -> Option<&'static str> { + Some("GLOB") + } + fn doc_category(&self) -> Category { + Category::Filter + } + fn doc_short(&self) -> &'static str { + r"Include or exclude file paths." + } + fn doc_long(&self) -> &'static str { + r#" +Include or exclude files and directories for searching that match the given +glob. This always overrides any other ignore logic. Multiple glob flags may +be used. Globbing rules match \fB.gitignore\fP globs. Precede a glob with a +\fB!\fP to exclude it. If multiple globs match a file or directory, the glob +given later in the command line takes precedence. +.sp +As an extension, globs support specifying alternatives: +.BI "\-g '" ab{c,d}* ' +is equivalent to +.BI "\-g " "abc " "\-g " abd. +Empty alternatives like +.BI "\-g '" ab{,c} ' +are not currently supported. Note that this syntax extension is also currently +enabled in \fBgitignore\fP files, even though this syntax isn't supported by +git itself. ripgrep may disable this syntax extension in gitignore files, but +it will always remain available via the \flag{glob} flag. +.sp +When this flag is set, every file and directory is applied to it to test for +a match. For example, if you only want to search in a particular directory +\fIfoo\fP, then +.BI "\-g " foo +is incorrect because \fIfoo/bar\fP does not match +the glob \fIfoo\fP. Instead, you should use +.BI "\-g '" foo/** '. +"# + } + + fn update(&self, v: FlagValue, args: &mut LowArgs) -> anyhow::Result<()> { + let glob = convert::string(v.unwrap_value())?; + args.globs.push(glob); + Ok(()) + } +} + +#[cfg(test)] +#[test] +fn test_glob() { + let args = parse_low_raw(None::<&str>).unwrap(); + assert_eq!(Vec::::new(), args.globs); + + let args = parse_low_raw(["--glob", "foo"]).unwrap(); + assert_eq!(vec!["foo".to_string()], args.globs); + + let args = parse_low_raw(["--glob=foo"]).unwrap(); + assert_eq!(vec!["foo".to_string()], args.globs); + + let args = parse_low_raw(["-g", "foo"]).unwrap(); + assert_eq!(vec!["foo".to_string()], args.globs); + + let args = parse_low_raw(["-gfoo"]).unwrap(); + assert_eq!(vec!["foo".to_string()], args.globs); + + let args = parse_low_raw(["--glob", "-foo"]).unwrap(); + assert_eq!(vec!["-foo".to_string()], args.globs); + + let args = parse_low_raw(["--glob=-foo"]).unwrap(); + assert_eq!(vec!["-foo".to_string()], args.globs); + + let args = parse_low_raw(["-g", "-foo"]).unwrap(); + assert_eq!(vec!["-foo".to_string()], args.globs); + + let args = parse_low_raw(["-g-foo"]).unwrap(); + assert_eq!(vec!["-foo".to_string()], args.globs); +} + +/// --glob-case-insensitive +#[derive(Debug)] +struct GlobCaseInsensitive; + +impl Flag for GlobCaseInsensitive { + fn is_switch(&self) -> bool { + true + } + fn name_long(&self) -> &'static str { + "glob-case-insensitive" + } + fn name_negated(&self) -> Option<&'static str> { + Some("no-glob-case-insensitive") + } + fn doc_category(&self) -> Category { + Category::Filter + } + fn doc_short(&self) -> &'static str { + r"Process all glob patterns case insensitively." + } + fn doc_long(&self) -> &'static str { + r" +Process all glob patterns given with the \flag{glob} flag case insensitively. +This effectively treats \flag{glob} as \flag{iglob}. +" + } + + fn update(&self, v: FlagValue, args: &mut LowArgs) -> anyhow::Result<()> { + args.glob_case_insensitive = v.unwrap_switch(); + Ok(()) + } +} + +#[cfg(test)] +#[test] +fn test_glob_case_insensitive() { + let args = parse_low_raw(None::<&str>).unwrap(); + assert_eq!(false, args.glob_case_insensitive); + + let args = parse_low_raw(["--glob-case-insensitive"]).unwrap(); + assert_eq!(true, args.glob_case_insensitive); + + let args = parse_low_raw([ + "--glob-case-insensitive", + "--no-glob-case-insensitive", + ]) + .unwrap(); + assert_eq!(false, args.glob_case_insensitive); + + let args = parse_low_raw([ + "--no-glob-case-insensitive", + "--glob-case-insensitive", + ]) + .unwrap(); + assert_eq!(true, args.glob_case_insensitive); +} + +/// --heading +#[derive(Debug)] +struct Heading; + +impl Flag for Heading { + fn is_switch(&self) -> bool { + true + } + fn name_long(&self) -> &'static str { + "heading" + } + fn name_negated(&self) -> Option<&'static str> { + Some("no-heading") + } + fn doc_category(&self) -> Category { + Category::Output + } + fn doc_short(&self) -> &'static str { + r"Print matches grouped by each file." + } + fn doc_long(&self) -> &'static str { + r" +This flag prints the file path above clusters of matches from each file instead +of printing the file path as a prefix for each matched line. +.sp +This is the default mode when printing to a tty. +.sp +When \fBstdout\fP is not a tty, then ripgrep will default to the standard +grep-like format. One can force this format in Unix-like environments by +piping the output of ripgrep to \fBcat\fP. For example, \fBrg\fP \fIfoo\fP \fB| +cat\fP. +" + } + + fn update(&self, v: FlagValue, args: &mut LowArgs) -> anyhow::Result<()> { + args.heading = Some(v.unwrap_switch()); + Ok(()) + } +} + +#[cfg(test)] +#[test] +fn test_heading() { + let args = parse_low_raw(None::<&str>).unwrap(); + assert_eq!(None, args.heading); + + let args = parse_low_raw(["--heading"]).unwrap(); + assert_eq!(Some(true), args.heading); + + let args = parse_low_raw(["--no-heading"]).unwrap(); + assert_eq!(Some(false), args.heading); + + let args = parse_low_raw(["--heading", "--no-heading"]).unwrap(); + assert_eq!(Some(false), args.heading); + + let args = parse_low_raw(["--no-heading", "--heading"]).unwrap(); + assert_eq!(Some(true), args.heading); +} + +/// -h/--help +#[derive(Debug)] +struct Help; + +impl Flag for Help { + fn is_switch(&self) -> bool { + true + } + fn name_long(&self) -> &'static str { + "help" + } + fn name_short(&self) -> Option { + Some(b'h') + } + fn doc_category(&self) -> Category { + Category::Output + } + fn doc_short(&self) -> &'static str { + r"Show help output." + } + fn doc_long(&self) -> &'static str { + r" +This flag prints the help output for ripgrep. +.sp +Unlike most other flags, the behavior of the short flag, \fB\-h\fP, and the +long flag, \fB\-\-help\fP, is different. The short flag will show a condensed +help output while the long flag will show a verbose help output. The verbose +help output has complete documentation, where as the condensed help output will +show only a single line for every flag. +" + } + + fn update(&self, v: FlagValue, _: &mut LowArgs) -> anyhow::Result<()> { + assert!(v.unwrap_switch(), "--help has no negation"); + // Since this flag has different semantics for -h and --help and the + // Flag trait doesn't support encoding this sort of thing, we handle it + // as a special case in the parser. + Ok(()) + } +} + +#[cfg(test)] +#[test] +fn test_help() { + let args = parse_low_raw(None::<&str>).unwrap(); + assert_eq!(None, args.special); + + let args = parse_low_raw(["-h"]).unwrap(); + assert_eq!(Some(SpecialMode::HelpShort), args.special); + + let args = parse_low_raw(["--help"]).unwrap(); + assert_eq!(Some(SpecialMode::HelpLong), args.special); + + let args = parse_low_raw(["-h", "--help"]).unwrap(); + assert_eq!(Some(SpecialMode::HelpLong), args.special); + + let args = parse_low_raw(["--help", "-h"]).unwrap(); + assert_eq!(Some(SpecialMode::HelpShort), args.special); +} + +/// -./--hidden +#[derive(Debug)] +struct Hidden; + +impl Flag for Hidden { + fn is_switch(&self) -> bool { + true + } + fn name_short(&self) -> Option { + Some(b'.') + } + fn name_long(&self) -> &'static str { + "hidden" + } + fn name_negated(&self) -> Option<&'static str> { + Some("no-hidden") + } + fn doc_category(&self) -> Category { + Category::Filter + } + fn doc_short(&self) -> &'static str { + r"Search hidden files and directories." + } + fn doc_long(&self) -> &'static str { + r#" +Search hidden files and directories. By default, hidden files and directories +are skipped. Note that if a hidden file or a directory is whitelisted in +an ignore file, then it will be searched even if this flag isn't provided. +Similarly if a hidden file or directory is given explicitly as an argument to +ripgrep. +.sp +A file or directory is considered hidden if its base name starts with a dot +character (\fB.\fP). On operating systems which support a "hidden" file +attribute, like Windows, files with this attribute are also considered hidden. +.sp +Note that \flag{hidden} will include files and folders like \fB.git\fP +regardless of \flag{no-ignore-vcs}. To exclude such paths when using +\flag{hidden}, you must explicitly ignore them using another flag or ignore +file. +"# + } + + fn update(&self, v: FlagValue, args: &mut LowArgs) -> anyhow::Result<()> { + args.hidden = v.unwrap_switch(); + Ok(()) + } +} + +#[cfg(test)] +#[test] +fn test_hidden() { + let args = parse_low_raw(None::<&str>).unwrap(); + assert_eq!(false, args.hidden); + + let args = parse_low_raw(["--hidden"]).unwrap(); + assert_eq!(true, args.hidden); + + let args = parse_low_raw(["-."]).unwrap(); + assert_eq!(true, args.hidden); + + let args = parse_low_raw(["-.", "--no-hidden"]).unwrap(); + assert_eq!(false, args.hidden); + + let args = parse_low_raw(["--no-hidden", "-."]).unwrap(); + assert_eq!(true, args.hidden); +} + +/// --hostname-bin +#[derive(Debug)] +struct HostnameBin; + +impl Flag for HostnameBin { + fn is_switch(&self) -> bool { + false + } + fn name_long(&self) -> &'static str { + "hostname-bin" + } + fn doc_variable(&self) -> Option<&'static str> { + Some("COMMAND") + } + fn doc_category(&self) -> Category { + Category::Output + } + fn doc_short(&self) -> &'static str { + r"Run a program to get this system's hostname." + } + fn doc_long(&self) -> &'static str { + r#" +This flag controls how ripgrep determines this system's hostname. The flag's +value should correspond to an executable (either a path or something that can +be found via your system's \fBPATH\fP environment variable). When set, ripgrep +will run this executable, with no arguments, and treat its output (with leading +and trailing whitespace stripped) as your system's hostname. +.sp +When not set (the default, or the empty string), ripgrep will try to +automatically detect your system's hostname. On Unix, this corresponds +to calling \fBgethostname\fP. On Windows, this corresponds to calling +\fBGetComputerNameExW\fP to fetch the system's "physical DNS hostname." +.sp +ripgrep uses your system's hostname for producing hyperlinks. +"# + } + fn completion_type(&self) -> CompletionType { + CompletionType::Executable + } + + fn update(&self, v: FlagValue, args: &mut LowArgs) -> anyhow::Result<()> { + let path = PathBuf::from(v.unwrap_value()); + args.hostname_bin = + if path.as_os_str().is_empty() { None } else { Some(path) }; + Ok(()) + } +} + +#[cfg(test)] +#[test] +fn test_hostname_bin() { + let args = parse_low_raw(None::<&str>).unwrap(); + assert_eq!(None, args.hostname_bin); + + let args = parse_low_raw(["--hostname-bin", "foo"]).unwrap(); + assert_eq!(Some(PathBuf::from("foo")), args.hostname_bin); + + let args = parse_low_raw(["--hostname-bin=foo"]).unwrap(); + assert_eq!(Some(PathBuf::from("foo")), args.hostname_bin); +} + +/// --hyperlink-format +#[derive(Debug)] +struct HyperlinkFormat; + +impl Flag for HyperlinkFormat { + fn is_switch(&self) -> bool { + false + } + fn name_long(&self) -> &'static str { + "hyperlink-format" + } + fn doc_variable(&self) -> Option<&'static str> { + Some("FORMAT") + } + fn doc_category(&self) -> Category { + Category::Output + } + fn doc_short(&self) -> &'static str { + r"Set the format of hyperlinks." + } + fn doc_long(&self) -> &'static str { + static DOC: LazyLock = LazyLock::new(|| { + let mut doc = String::new(); + doc.push_str( + r#" +Set the format of hyperlinks to use when printing results. Hyperlinks make +certain elements of ripgrep's output, such as file paths, clickable. This +generally only works in terminal emulators that support OSC-8 hyperlinks. For +example, the format \fBfile://{host}{path}\fP will emit an RFC 8089 hyperlink. +To see the format that ripgrep is using, pass the \flag{debug} flag. +.sp +Alternatively, a format string may correspond to one of the following aliases: +"#, + ); + + let mut aliases = grep::printer::hyperlink_aliases(); + aliases.sort_by_key(|alias| { + alias.display_priority().unwrap_or(i16::MAX) + }); + for (i, alias) in aliases.iter().enumerate() { + doc.push_str(r"\fB"); + doc.push_str(alias.name()); + doc.push_str(r"\fP"); + doc.push_str(if i < aliases.len() - 1 { ", " } else { "." }); + } + doc.push_str( + r#" +The alias will be replaced with a format string that is intended to work for +the corresponding application. +.sp +The following variables are available in the format string: +.sp +.TP 12 +\fB{path}\fP +Required. This is replaced with a path to a matching file. The path is +guaranteed to be absolute and percent encoded such that it is valid to put into +a URI. Note that a path is guaranteed to start with a /. +.TP 12 +\fB{host}\fP +Optional. This is replaced with your system's hostname. On Unix, this +corresponds to calling \fBgethostname\fP. On Windows, this corresponds to +calling \fBGetComputerNameExW\fP to fetch the system's "physical DNS hostname." +Alternatively, if \flag{hostname-bin} was provided, then the hostname returned +from the output of that program will be returned. If no hostname could be +found, then this variable is replaced with the empty string. +.TP 12 +\fB{line}\fP +Optional. If appropriate, this is replaced with the line number of a match. If +no line number is available (for example, if \fB\-\-no\-line\-number\fP was +given), then it is automatically replaced with the value 1. +.TP 12 +\fB{column}\fP +Optional, but requires the presence of \fB{line}\fP. If appropriate, this is +replaced with the column number of a match. If no column number is available +(for example, if \fB\-\-no\-column\fP was given), then it is automatically +replaced with the value 1. +.TP 12 +\fB{wslprefix}\fP +Optional. This is a special value that is set to +\fBwsl$/\fP\fIWSL_DISTRO_NAME\fP, where \fIWSL_DISTRO_NAME\fP corresponds to +the value of the equivalent environment variable. If the system is not Unix +or if the \fIWSL_DISTRO_NAME\fP environment variable is not set, then this is +replaced with the empty string. +.PP +A format string may be empty. An empty format string is equivalent to the +\fBnone\fP alias. In this case, hyperlinks will be disabled. +.sp +At present, ripgrep does not enable hyperlinks by default. Users must opt into +them. If you aren't sure what format to use, try \fBdefault\fP. +.sp +Like colors, when ripgrep detects that stdout is not connected to a tty, then +hyperlinks are automatically disabled, regardless of the value of this flag. +Users can pass \fB\-\-color=always\fP to forcefully emit hyperlinks. +.sp +Note that hyperlinks are only written when a path is also in the output +and colors are enabled. To write hyperlinks without colors, you'll need to +configure ripgrep to not colorize anything without actually disabling all ANSI +escape codes completely: +.sp +.EX + \-\-colors 'path:none' \\ + \-\-colors 'line:none' \\ + \-\-colors 'column:none' \\ + \-\-colors 'match:none' +.EE +.sp +ripgrep works this way because it treats the \flag{color} flag as a proxy for +whether ANSI escape codes should be used at all. This means that environment +variables like \fBNO_COLOR=1\fP and \fBTERM=dumb\fP not only disable colors, +but hyperlinks as well. Similarly, colors and hyperlinks are disabled when +ripgrep is not writing to a tty. (Unless one forces the issue by setting +\fB\-\-color=always\fP.) +.sp +If you're searching a file directly, for example: +.sp +.EX + rg foo path/to/file +.EE +.sp +then hyperlinks will not be emitted since the path given does not appear +in the output. To make the path appear, and thus also a hyperlink, use the +\flag{with-filename} flag. +.sp +For more information on hyperlinks in terminal emulators, see: +https://gist.github.com/egmontkob/eb114294efbcd5adb1944c9f3cb5feda +"#, + ); + doc + }); + &DOC + } + + fn doc_choices(&self) -> &'static [&'static str] { + static CHOICES: LazyLock> = LazyLock::new(|| { + let mut aliases = grep::printer::hyperlink_aliases(); + aliases.sort_by_key(|alias| { + alias.display_priority().unwrap_or(i16::MAX) + }); + aliases.iter().map(|alias| alias.name().to_string()).collect() + }); + static BORROWED: LazyLock> = + LazyLock::new(|| CHOICES.iter().map(|name| &**name).collect()); + &*BORROWED + } + + fn update(&self, v: FlagValue, args: &mut LowArgs) -> anyhow::Result<()> { + let v = v.unwrap_value(); + let string = convert::str(&v)?; + let format = string.parse().context("invalid hyperlink format")?; + args.hyperlink_format = format; + Ok(()) + } +} + +#[cfg(test)] +#[test] +fn test_hyperlink_format() { + let parseformat = |format: &str| { + format.parse::().unwrap() + }; + + let args = parse_low_raw(None::<&str>).unwrap(); + assert_eq!(parseformat("none"), args.hyperlink_format); + + let args = parse_low_raw(["--hyperlink-format", "default"]).unwrap(); + #[cfg(windows)] + assert_eq!(parseformat("file://{path}"), args.hyperlink_format); + #[cfg(not(windows))] + assert_eq!(parseformat("file://{host}{path}"), args.hyperlink_format); + + let args = parse_low_raw(["--hyperlink-format", "file"]).unwrap(); + assert_eq!(parseformat("file://{host}{path}"), args.hyperlink_format); + + let args = parse_low_raw([ + "--hyperlink-format", + "file", + "--hyperlink-format=grep+", + ]) + .unwrap(); + assert_eq!(parseformat("grep+://{path}:{line}"), args.hyperlink_format); + + let args = + parse_low_raw(["--hyperlink-format", "file://{host}{path}#{line}"]) + .unwrap(); + assert_eq!( + parseformat("file://{host}{path}#{line}"), + args.hyperlink_format + ); + + let result = parse_low_raw(["--hyperlink-format", "file://heythere"]); + assert!(result.is_err(), "{result:?}"); +} + +/// --iglob +#[derive(Debug)] +struct IGlob; + +impl Flag for IGlob { + fn is_switch(&self) -> bool { + false + } + fn name_long(&self) -> &'static str { + "iglob" + } + fn doc_variable(&self) -> Option<&'static str> { + Some("GLOB") + } + fn doc_category(&self) -> Category { + Category::Filter + } + fn doc_short(&self) -> &'static str { + r"Include/exclude paths case insensitively." + } + fn doc_long(&self) -> &'static str { + r" +Include or exclude files and directories for searching that match the given +glob. This always overrides any other ignore logic. Multiple glob flags may +be used. Globbing rules match \fB.gitignore\fP globs. Precede a glob with a +\fB!\fP to exclude it. If multiple globs match a file or directory, the glob +given later in the command line takes precedence. Globs used via this flag are +matched case insensitively. +" + } + + fn update(&self, v: FlagValue, args: &mut LowArgs) -> anyhow::Result<()> { + let glob = convert::string(v.unwrap_value())?; + args.iglobs.push(glob); + Ok(()) + } +} + +#[cfg(test)] +#[test] +fn test_iglob() { + let args = parse_low_raw(None::<&str>).unwrap(); + assert_eq!(Vec::::new(), args.iglobs); + + let args = parse_low_raw(["--iglob", "foo"]).unwrap(); + assert_eq!(vec!["foo".to_string()], args.iglobs); + + let args = parse_low_raw(["--iglob=foo"]).unwrap(); + assert_eq!(vec!["foo".to_string()], args.iglobs); + + let args = parse_low_raw(["--iglob", "-foo"]).unwrap(); + assert_eq!(vec!["-foo".to_string()], args.iglobs); + + let args = parse_low_raw(["--iglob=-foo"]).unwrap(); + assert_eq!(vec!["-foo".to_string()], args.iglobs); +} + +/// -i/--ignore-case +#[derive(Debug)] +struct IgnoreCase; + +impl Flag for IgnoreCase { + fn is_switch(&self) -> bool { + true + } + fn name_short(&self) -> Option { + Some(b'i') + } + fn name_long(&self) -> &'static str { + "ignore-case" + } + fn doc_category(&self) -> Category { + Category::Search + } + fn doc_short(&self) -> &'static str { + r"Case insensitive search." + } + fn doc_long(&self) -> &'static str { + r#" +When this flag is provided, all patterns will be searched case insensitively. +The case insensitivity rules used by ripgrep's default regex engine conform to +Unicode's "simple" case folding rules. +.sp +This is a global option that applies to all patterns given to ripgrep. +Individual patterns can still be matched case sensitively by using +inline regex flags. For example, \fB(?\-i)abc\fP will match \fBabc\fP +case sensitively even when this flag is used. +.sp +This flag overrides \flag{case-sensitive} and \flag{smart-case}. +"# + } + + fn update(&self, v: FlagValue, args: &mut LowArgs) -> anyhow::Result<()> { + assert!(v.unwrap_switch(), "flag has no negation"); + args.case = CaseMode::Insensitive; + Ok(()) + } +} + +#[cfg(test)] +#[test] +fn test_ignore_case() { + let args = parse_low_raw(None::<&str>).unwrap(); + assert_eq!(CaseMode::Sensitive, args.case); + + let args = parse_low_raw(["--ignore-case"]).unwrap(); + assert_eq!(CaseMode::Insensitive, args.case); + + let args = parse_low_raw(["-i"]).unwrap(); + assert_eq!(CaseMode::Insensitive, args.case); + + let args = parse_low_raw(["-i", "-s"]).unwrap(); + assert_eq!(CaseMode::Sensitive, args.case); + + let args = parse_low_raw(["-s", "-i"]).unwrap(); + assert_eq!(CaseMode::Insensitive, args.case); +} + +/// --ignore-file +#[derive(Debug)] +struct IgnoreFile; + +impl Flag for IgnoreFile { + fn is_switch(&self) -> bool { + false + } + fn name_long(&self) -> &'static str { + "ignore-file" + } + fn doc_variable(&self) -> Option<&'static str> { + Some("PATH") + } + fn doc_category(&self) -> Category { + Category::Filter + } + fn doc_short(&self) -> &'static str { + r"Specify additional ignore files." + } + fn doc_long(&self) -> &'static str { + r" +Specifies a path to one or more \fBgitignore\fP formatted rules files. +These patterns are applied after the patterns found in \fB.gitignore\fP, +\fB.rgignore\fP and \fB.ignore\fP are applied and are matched relative to the +current working directory. That is, files specified via this flag have lower +precedence than files automatically found in the directory tree. Multiple +additional ignore files can be specified by using this flag repeatedly. When +specifying multiple ignore files, earlier files have lower precedence than +later files. +.sp +If you are looking for a way to include or exclude files and directories +directly on the command line, then use \flag{glob} instead. +" + } + fn completion_type(&self) -> CompletionType { + CompletionType::Filename + } + + fn update(&self, v: FlagValue, args: &mut LowArgs) -> anyhow::Result<()> { + let path = PathBuf::from(v.unwrap_value()); + args.ignore_file.push(path); + Ok(()) + } +} + +#[cfg(test)] +#[test] +fn test_ignore_file() { + let args = parse_low_raw(None::<&str>).unwrap(); + assert_eq!(Vec::::new(), args.ignore_file); + + let args = parse_low_raw(["--ignore-file", "foo"]).unwrap(); + assert_eq!(vec![PathBuf::from("foo")], args.ignore_file); + + let args = parse_low_raw(["--ignore-file", "foo", "--ignore-file", "bar"]) + .unwrap(); + assert_eq!( + vec![PathBuf::from("foo"), PathBuf::from("bar")], + args.ignore_file + ); +} + +/// --ignore-file-case-insensitive +#[derive(Debug)] +struct IgnoreFileCaseInsensitive; + +impl Flag for IgnoreFileCaseInsensitive { + fn is_switch(&self) -> bool { + true + } + fn name_long(&self) -> &'static str { + "ignore-file-case-insensitive" + } + fn name_negated(&self) -> Option<&'static str> { + Some("no-ignore-file-case-insensitive") + } + fn doc_category(&self) -> Category { + Category::Filter + } + fn doc_short(&self) -> &'static str { + r"Process ignore files case insensitively." + } + fn doc_long(&self) -> &'static str { + r" +Process ignore files (\fB.gitignore\fP, \fB.ignore\fP, etc.) case +insensitively. Note that this comes with a performance penalty and is most +useful on case insensitive file systems (such as Windows). +" + } + + fn update(&self, v: FlagValue, args: &mut LowArgs) -> anyhow::Result<()> { + args.ignore_file_case_insensitive = v.unwrap_switch(); + Ok(()) + } +} + +#[cfg(test)] +#[test] +fn test_ignore_file_case_insensitive() { + let args = parse_low_raw(None::<&str>).unwrap(); + assert_eq!(false, args.ignore_file_case_insensitive); + + let args = parse_low_raw(["--ignore-file-case-insensitive"]).unwrap(); + assert_eq!(true, args.ignore_file_case_insensitive); + + let args = parse_low_raw([ + "--ignore-file-case-insensitive", + "--no-ignore-file-case-insensitive", + ]) + .unwrap(); + assert_eq!(false, args.ignore_file_case_insensitive); + + let args = parse_low_raw([ + "--no-ignore-file-case-insensitive", + "--ignore-file-case-insensitive", + ]) + .unwrap(); + assert_eq!(true, args.ignore_file_case_insensitive); +} + +/// --include-zero +#[derive(Debug)] +struct IncludeZero; + +impl Flag for IncludeZero { + fn is_switch(&self) -> bool { + true + } + fn name_long(&self) -> &'static str { + "include-zero" + } + fn name_negated(&self) -> Option<&'static str> { + Some("no-include-zero") + } + fn doc_category(&self) -> Category { + Category::Output + } + fn doc_short(&self) -> &'static str { + r"Include zero matches in summary output." + } + fn doc_long(&self) -> &'static str { + r" +When used with \flag{count} or \flag{count-matches}, this causes ripgrep to +print the number of matches for each file even if there were zero matches. This +is disabled by default but can be enabled to make ripgrep behave more like +grep. +" + } + + fn update(&self, v: FlagValue, args: &mut LowArgs) -> anyhow::Result<()> { + args.include_zero = v.unwrap_switch(); + Ok(()) + } +} + +#[cfg(test)] +#[test] +fn test_include_zero() { + let args = parse_low_raw(None::<&str>).unwrap(); + assert_eq!(false, args.include_zero); + + let args = parse_low_raw(["--include-zero"]).unwrap(); + assert_eq!(true, args.include_zero); + + let args = parse_low_raw(["--include-zero", "--no-include-zero"]).unwrap(); + assert_eq!(false, args.include_zero); +} + +/// -v/--invert-match +#[derive(Debug)] +struct InvertMatch; + +impl Flag for InvertMatch { + fn is_switch(&self) -> bool { + true + } + fn name_short(&self) -> Option { + Some(b'v') + } + fn name_long(&self) -> &'static str { + "invert-match" + } + fn name_negated(&self) -> Option<&'static str> { + Some("no-invert-match") + } + fn doc_category(&self) -> Category { + Category::Search + } + fn doc_short(&self) -> &'static str { + r"Invert matching." + } + fn doc_long(&self) -> &'static str { + r" +This flag inverts matching. That is, instead of printing lines that match, +ripgrep will print lines that don't match. +.sp +Note that this only inverts line-by-line matching. For example, combining this +flag with \flag{files-with-matches} will emit files that contain any lines +that do not match the patterns given. That's not the same as, for example, +\flag{files-without-match}, which will emit files that do not contain any +matching lines. +" + } + + fn update(&self, v: FlagValue, args: &mut LowArgs) -> anyhow::Result<()> { + args.invert_match = v.unwrap_switch(); + Ok(()) + } +} + +#[cfg(test)] +#[test] +fn test_invert_match() { + let args = parse_low_raw(None::<&str>).unwrap(); + assert_eq!(false, args.invert_match); + + let args = parse_low_raw(["--invert-match"]).unwrap(); + assert_eq!(true, args.invert_match); + + let args = parse_low_raw(["-v"]).unwrap(); + assert_eq!(true, args.invert_match); + + let args = parse_low_raw(["-v", "--no-invert-match"]).unwrap(); + assert_eq!(false, args.invert_match); +} + +/// --json +#[derive(Debug)] +struct JSON; + +impl Flag for JSON { + fn is_switch(&self) -> bool { + true + } + fn name_long(&self) -> &'static str { + "json" + } + fn name_negated(&self) -> Option<&'static str> { + Some("no-json") + } + fn doc_category(&self) -> Category { + Category::OutputModes + } + fn doc_short(&self) -> &'static str { + r"Show search results in a JSON Lines format." + } + fn doc_long(&self) -> &'static str { + r" +Enable printing results in a JSON Lines format. +.sp +When this flag is provided, ripgrep will emit a sequence of messages, each +encoded as a JSON object, where there are five different message types: +.sp +.TP 12 +\fBbegin\fP +A message that indicates a file is being searched and contains at least one +match. +.TP 12 +\fBend\fP +A message the indicates a file is done being searched. This message also +include summary statistics about the search for a particular file. +.TP 12 +\fBmatch\fP +A message that indicates a match was found. This includes the text and offsets +of the match. +.TP 12 +\fBcontext\fP +A message that indicates a contextual line was found. This includes the text of +the line, along with any match information if the search was inverted. +.TP 12 +\fBsummary\fP +The final message emitted by ripgrep that contains summary statistics about the +search across all files. +.PP +Since file paths or the contents of files are not guaranteed to be valid +UTF-8 and JSON itself must be representable by a Unicode encoding, ripgrep +will emit all data elements as objects with one of two keys: \fBtext\fP or +\fBbytes\fP. \fBtext\fP is a normal JSON string when the data is valid UTF-8 +while \fBbytes\fP is the base64 encoded contents of the data. +.sp +The JSON Lines format is only supported for showing search results. It cannot +be used with other flags that emit other types of output, such as \flag{files}, +\flag{files-with-matches}, \flag{files-without-match}, \flag{count} or +\flag{count-matches}. ripgrep will report an error if any of the aforementioned +flags are used in concert with \flag{json}. +.sp +Other flags that control aspects of the standard output such as +\flag{only-matching}, \flag{heading}, \flag{replace}, \flag{max-columns}, etc., +have no effect when \flag{json} is set. However, enabling JSON output will +always implicitly and unconditionally enable \flag{stats}. +.sp +A more complete description of the JSON format used can be found here: +\fIhttps://docs.rs/grep-printer/*/grep_printer/struct.JSON.html\fP. +" + } + + fn update(&self, v: FlagValue, args: &mut LowArgs) -> anyhow::Result<()> { + if v.unwrap_switch() { + args.mode.update(Mode::Search(SearchMode::JSON)); + } else if matches!(args.mode, Mode::Search(SearchMode::JSON)) { + // --no-json only reverts to the default mode if the mode is + // JSON, otherwise it's a no-op. + args.mode.update(Mode::Search(SearchMode::Standard)); + } + Ok(()) + } +} + +#[cfg(test)] +#[test] +fn test_json() { + let args = parse_low_raw(None::<&str>).unwrap(); + assert_eq!(Mode::Search(SearchMode::Standard), args.mode); + + let args = parse_low_raw(["--json"]).unwrap(); + assert_eq!(Mode::Search(SearchMode::JSON), args.mode); + + let args = parse_low_raw(["--json", "--no-json"]).unwrap(); + assert_eq!(Mode::Search(SearchMode::Standard), args.mode); + + let args = parse_low_raw(["--json", "--files", "--no-json"]).unwrap(); + assert_eq!(Mode::Files, args.mode); + + let args = parse_low_raw(["--json", "-l", "--no-json"]).unwrap(); + assert_eq!(Mode::Search(SearchMode::FilesWithMatches), args.mode); +} + +/// --line-buffered +#[derive(Debug)] +struct LineBuffered; + +impl Flag for LineBuffered { + fn is_switch(&self) -> bool { + true + } + fn name_long(&self) -> &'static str { + "line-buffered" + } + fn name_negated(&self) -> Option<&'static str> { + Some("no-line-buffered") + } + fn doc_category(&self) -> Category { + Category::Output + } + fn doc_short(&self) -> &'static str { + r"Force line buffering." + } + fn doc_long(&self) -> &'static str { + r" +When enabled, ripgrep will always use line buffering. That is, whenever a +matching line is found, it will be flushed to stdout immediately. This is the +default when ripgrep's stdout is connected to a tty, but otherwise, ripgrep +will use block buffering, which is typically faster. This flag forces ripgrep +to use line buffering even if it would otherwise use block buffering. This is +typically useful in shell pipelines, for example: +.sp +.EX + tail -f something.log | rg foo --line-buffered | rg bar +.EE +.sp +This overrides the \flag{block-buffered} flag. +" + } + + fn update(&self, v: FlagValue, args: &mut LowArgs) -> anyhow::Result<()> { + args.buffer = if v.unwrap_switch() { + BufferMode::Line + } else { + BufferMode::Auto + }; + Ok(()) + } +} + +#[cfg(test)] +#[test] +fn test_line_buffered() { + let args = parse_low_raw(None::<&str>).unwrap(); + assert_eq!(BufferMode::Auto, args.buffer); + + let args = parse_low_raw(["--line-buffered"]).unwrap(); + assert_eq!(BufferMode::Line, args.buffer); + + let args = + parse_low_raw(["--line-buffered", "--no-line-buffered"]).unwrap(); + assert_eq!(BufferMode::Auto, args.buffer); + + let args = parse_low_raw(["--line-buffered", "--block-buffered"]).unwrap(); + assert_eq!(BufferMode::Block, args.buffer); +} + +/// -n/--line-number +#[derive(Debug)] +struct LineNumber; + +impl Flag for LineNumber { + fn is_switch(&self) -> bool { + true + } + fn name_short(&self) -> Option { + Some(b'n') + } + fn name_long(&self) -> &'static str { + "line-number" + } + fn doc_category(&self) -> Category { + Category::Output + } + fn doc_short(&self) -> &'static str { + r"Show line numbers." + } + fn doc_long(&self) -> &'static str { + r" +Show line numbers (1-based). +.sp +This is enabled by default when stdout is connected to a tty. +.sp +This flag can be disabled by \flag{no-line-number}. +" + } + + fn update(&self, v: FlagValue, args: &mut LowArgs) -> anyhow::Result<()> { + assert!(v.unwrap_switch(), "--line-number has no automatic negation"); + args.line_number = Some(true); + Ok(()) + } +} + +#[cfg(test)] +#[test] +fn test_line_number() { + let args = parse_low_raw(None::<&str>).unwrap(); + assert_eq!(None, args.line_number); + + let args = parse_low_raw(["--line-number"]).unwrap(); + assert_eq!(Some(true), args.line_number); + + let args = parse_low_raw(["-n"]).unwrap(); + assert_eq!(Some(true), args.line_number); + + let args = parse_low_raw(["-n", "--no-line-number"]).unwrap(); + assert_eq!(Some(false), args.line_number); +} + +/// -N/--no-line-number +#[derive(Debug)] +struct LineNumberNo; + +impl Flag for LineNumberNo { + fn is_switch(&self) -> bool { + true + } + fn name_short(&self) -> Option { + Some(b'N') + } + fn name_long(&self) -> &'static str { + "no-line-number" + } + fn doc_category(&self) -> Category { + Category::Output + } + fn doc_short(&self) -> &'static str { + r"Suppress line numbers." + } + fn doc_long(&self) -> &'static str { + r" +Suppress line numbers. +.sp +Line numbers are off by default when stdout is not connected to a tty. +.sp +Line numbers can be forcefully turned on by \flag{line-number}. +" + } + + fn update(&self, v: FlagValue, args: &mut LowArgs) -> anyhow::Result<()> { + assert!( + v.unwrap_switch(), + "--no-line-number has no automatic negation" + ); + args.line_number = Some(false); + Ok(()) + } +} + +#[cfg(test)] +#[test] +fn test_no_line_number() { + let args = parse_low_raw(None::<&str>).unwrap(); + assert_eq!(None, args.line_number); + + let args = parse_low_raw(["--no-line-number"]).unwrap(); + assert_eq!(Some(false), args.line_number); + + let args = parse_low_raw(["-N"]).unwrap(); + assert_eq!(Some(false), args.line_number); + + let args = parse_low_raw(["-N", "--line-number"]).unwrap(); + assert_eq!(Some(true), args.line_number); +} + +/// -x/--line-regexp +#[derive(Debug)] +struct LineRegexp; + +impl Flag for LineRegexp { + fn is_switch(&self) -> bool { + true + } + fn name_short(&self) -> Option { + Some(b'x') + } + fn name_long(&self) -> &'static str { + "line-regexp" + } + fn doc_category(&self) -> Category { + Category::Search + } + fn doc_short(&self) -> &'static str { + r"Show matches surrounded by line boundaries." + } + fn doc_long(&self) -> &'static str { + r" +When enabled, ripgrep will only show matches surrounded by line boundaries. +This is equivalent to surrounding every pattern with \fB^\fP and \fB$\fP. In +other words, this only prints lines where the entire line participates in a +match. +.sp +This overrides the \flag{word-regexp} flag. +" + } + + fn update(&self, v: FlagValue, args: &mut LowArgs) -> anyhow::Result<()> { + assert!(v.unwrap_switch(), "--line-regexp has no negation"); + args.boundary = Some(BoundaryMode::Line); + Ok(()) + } +} + +#[cfg(test)] +#[test] +fn test_line_regexp() { + let args = parse_low_raw(None::<&str>).unwrap(); + assert_eq!(None, args.boundary); + + let args = parse_low_raw(["--line-regexp"]).unwrap(); + assert_eq!(Some(BoundaryMode::Line), args.boundary); + + let args = parse_low_raw(["-x"]).unwrap(); + assert_eq!(Some(BoundaryMode::Line), args.boundary); +} + +/// -M/--max-columns +#[derive(Debug)] +struct MaxColumns; + +impl Flag for MaxColumns { + fn is_switch(&self) -> bool { + false + } + fn name_short(&self) -> Option { + Some(b'M') + } + fn name_long(&self) -> &'static str { + "max-columns" + } + fn doc_variable(&self) -> Option<&'static str> { + Some("NUM") + } + fn doc_category(&self) -> Category { + Category::Output + } + fn doc_short(&self) -> &'static str { + r"Omit lines longer than this limit." + } + fn doc_long(&self) -> &'static str { + r" +When given, ripgrep will omit lines longer than this limit in bytes. Instead of +printing long lines, only the number of matches in that line is printed. +.sp +When this flag is omitted or is set to \fB0\fP, then it has no effect. +" + } + + fn update(&self, v: FlagValue, args: &mut LowArgs) -> anyhow::Result<()> { + let max = convert::u64(&v.unwrap_value())?; + args.max_columns = if max == 0 { None } else { Some(max) }; + Ok(()) + } +} + +#[cfg(test)] +#[test] +fn test_max_columns() { + let args = parse_low_raw(None::<&str>).unwrap(); + assert_eq!(None, args.max_columns); + + let args = parse_low_raw(["--max-columns", "5"]).unwrap(); + assert_eq!(Some(5), args.max_columns); + + let args = parse_low_raw(["-M", "5"]).unwrap(); + assert_eq!(Some(5), args.max_columns); + + let args = parse_low_raw(["-M5"]).unwrap(); + assert_eq!(Some(5), args.max_columns); + + let args = parse_low_raw(["--max-columns", "5", "-M0"]).unwrap(); + assert_eq!(None, args.max_columns); +} + +/// --max-columns-preview +#[derive(Debug)] +struct MaxColumnsPreview; + +impl Flag for MaxColumnsPreview { + fn is_switch(&self) -> bool { + true + } + fn name_long(&self) -> &'static str { + "max-columns-preview" + } + fn name_negated(&self) -> Option<&'static str> { + Some("no-max-columns-preview") + } + fn doc_category(&self) -> Category { + Category::Output + } + fn doc_short(&self) -> &'static str { + r"Show preview for lines exceeding the limit." + } + fn doc_long(&self) -> &'static str { + r" +Prints a preview for lines exceeding the configured max column limit. +.sp +When the \flag{max-columns} flag is used, ripgrep will by default completely +replace any line that is too long with a message indicating that a matching +line was removed. When this flag is combined with \flag{max-columns}, a preview +of the line (corresponding to the limit size) is shown instead, where the part +of the line exceeding the limit is not shown. +.sp +If the \flag{max-columns} flag is not set, then this has no effect. +" + } + + fn update(&self, v: FlagValue, args: &mut LowArgs) -> anyhow::Result<()> { + args.max_columns_preview = v.unwrap_switch(); + Ok(()) + } +} + +#[cfg(test)] +#[test] +fn test_max_columns_preview() { + let args = parse_low_raw(None::<&str>).unwrap(); + assert_eq!(false, args.max_columns_preview); + + let args = parse_low_raw(["--max-columns-preview"]).unwrap(); + assert_eq!(true, args.max_columns_preview); + + let args = + parse_low_raw(["--max-columns-preview", "--no-max-columns-preview"]) + .unwrap(); + assert_eq!(false, args.max_columns_preview); +} + +/// -m/--max-count +#[derive(Debug)] +struct MaxCount; + +impl Flag for MaxCount { + fn is_switch(&self) -> bool { + false + } + fn name_short(&self) -> Option { + Some(b'm') + } + fn name_long(&self) -> &'static str { + "max-count" + } + fn doc_variable(&self) -> Option<&'static str> { + Some("NUM") + } + fn doc_category(&self) -> Category { + Category::Search + } + fn doc_short(&self) -> &'static str { + r"Limit the number of matching lines." + } + fn doc_long(&self) -> &'static str { + r" +Limit the number of matching lines per file searched to \fINUM\fP. +.sp +When \flag{multiline} is used, a single match that spans multiple lines is only +counted once for the purposes of this limit. Multiple matches in a single line +are counted only once, as they would be in non-multiline mode. +.sp +When combined with \flag{after-context} or \flag{context}, it's possible for +more matches than the maximum to be printed if contextual lines contain a +match. +.sp +Note that \fB0\fP is a legal value but not likely to be useful. When used, +ripgrep won't search anything. +" + } + + fn update(&self, v: FlagValue, args: &mut LowArgs) -> anyhow::Result<()> { + args.max_count = Some(convert::u64(&v.unwrap_value())?); + Ok(()) + } +} + +#[cfg(test)] +#[test] +fn test_max_count() { + let args = parse_low_raw(None::<&str>).unwrap(); + assert_eq!(None, args.max_count); + + let args = parse_low_raw(["--max-count", "5"]).unwrap(); + assert_eq!(Some(5), args.max_count); + + let args = parse_low_raw(["-m", "5"]).unwrap(); + assert_eq!(Some(5), args.max_count); + + let args = parse_low_raw(["-m", "5", "--max-count=10"]).unwrap(); + assert_eq!(Some(10), args.max_count); + let args = parse_low_raw(["-m0"]).unwrap(); + assert_eq!(Some(0), args.max_count); +} + +/// --max-depth +#[derive(Debug)] +struct MaxDepth; + +impl Flag for MaxDepth { + fn is_switch(&self) -> bool { + false + } + fn name_short(&self) -> Option { + Some(b'd') + } + fn name_long(&self) -> &'static str { + "max-depth" + } + fn aliases(&self) -> &'static [&'static str] { + &["maxdepth"] + } + fn doc_variable(&self) -> Option<&'static str> { + Some("NUM") + } + fn doc_category(&self) -> Category { + Category::Filter + } + fn doc_short(&self) -> &'static str { + r"Descend at most NUM directories." + } + fn doc_long(&self) -> &'static str { + r" +This flag limits the depth of directory traversal to \fINUM\fP levels beyond +the paths given. A value of \fB0\fP only searches the explicitly given paths +themselves. +.sp +For example, \fBrg --max-depth 0 \fP\fIdir/\fP is a no-op because \fIdir/\fP +will not be descended into. \fBrg --max-depth 1 \fP\fIdir/\fP will search only +the direct children of \fIdir\fP. +.sp +An alternative spelling for this flag is \fB\-\-maxdepth\fP. +" + } + + fn update(&self, v: FlagValue, args: &mut LowArgs) -> anyhow::Result<()> { + args.max_depth = Some(convert::usize(&v.unwrap_value())?); + Ok(()) + } +} + +#[cfg(test)] +#[test] +fn test_max_depth() { + let args = parse_low_raw(None::<&str>).unwrap(); + assert_eq!(None, args.max_depth); + + let args = parse_low_raw(["--max-depth", "5"]).unwrap(); + assert_eq!(Some(5), args.max_depth); + + let args = parse_low_raw(["-d", "5"]).unwrap(); + assert_eq!(Some(5), args.max_depth); + + let args = parse_low_raw(["--max-depth", "5", "--max-depth=10"]).unwrap(); + assert_eq!(Some(10), args.max_depth); + + let args = parse_low_raw(["--max-depth", "0"]).unwrap(); + assert_eq!(Some(0), args.max_depth); + + let args = parse_low_raw(["--maxdepth", "5"]).unwrap(); + assert_eq!(Some(5), args.max_depth); +} + +/// --max-filesize +#[derive(Debug)] +struct MaxFilesize; + +impl Flag for MaxFilesize { + fn is_switch(&self) -> bool { + false + } + fn name_long(&self) -> &'static str { + "max-filesize" + } + fn doc_variable(&self) -> Option<&'static str> { + Some("NUM+SUFFIX?") + } + fn doc_category(&self) -> Category { + Category::Filter + } + fn doc_short(&self) -> &'static str { + r"Ignore files larger than NUM in size." + } + fn doc_long(&self) -> &'static str { + r" +Ignore files larger than \fINUM\fP in size. This does not apply to directories. +.sp +The input format accepts suffixes of \fBK\fP, \fBM\fP or \fBG\fP which +correspond to kilobytes, megabytes and gigabytes, respectively. If no suffix is +provided the input is treated as bytes. +.sp +Examples: \fB\-\-max-filesize 50K\fP or \fB\-\-max\-filesize 80M\fP. +" + } + + fn update(&self, v: FlagValue, args: &mut LowArgs) -> anyhow::Result<()> { + let v = v.unwrap_value(); + args.max_filesize = Some(convert::human_readable_u64(&v)?); + Ok(()) + } +} + +#[cfg(test)] +#[test] +fn test_max_filesize() { + let args = parse_low_raw(None::<&str>).unwrap(); + assert_eq!(None, args.max_filesize); + + let args = parse_low_raw(["--max-filesize", "1024"]).unwrap(); + assert_eq!(Some(1024), args.max_filesize); + + let args = parse_low_raw(["--max-filesize", "1K"]).unwrap(); + assert_eq!(Some(1024), args.max_filesize); + + let args = + parse_low_raw(["--max-filesize", "1K", "--max-filesize=1M"]).unwrap(); + assert_eq!(Some(1024 * 1024), args.max_filesize); +} + +/// --mmap +#[derive(Debug)] +struct Mmap; + +impl Flag for Mmap { + fn is_switch(&self) -> bool { + true + } + fn name_long(&self) -> &'static str { + "mmap" + } + fn name_negated(&self) -> Option<&'static str> { + Some("no-mmap") + } + fn doc_category(&self) -> Category { + Category::Search + } + fn doc_short(&self) -> &'static str { + r"Search with memory maps when possible." + } + fn doc_long(&self) -> &'static str { + r" +When enabled, ripgrep will search using memory maps when possible. This is +enabled by default when ripgrep thinks it will be faster. +.sp +Memory map searching cannot be used in all circumstances. For example, when +searching virtual files or streams likes \fBstdin\fP. In such cases, memory +maps will not be used even when this flag is enabled. +.sp +Note that ripgrep may abort unexpectedly when memory maps are used if it +searches a file that is simultaneously truncated. Users can opt out of this +possibility by disabling memory maps. +" + } + + fn update(&self, v: FlagValue, args: &mut LowArgs) -> anyhow::Result<()> { + args.mmap = if v.unwrap_switch() { + MmapMode::AlwaysTryMmap + } else { + MmapMode::Never + }; + Ok(()) + } +} + +#[cfg(test)] +#[test] +fn test_mmap() { + let args = parse_low_raw(None::<&str>).unwrap(); + assert_eq!(MmapMode::Auto, args.mmap); + + let args = parse_low_raw(["--mmap"]).unwrap(); + assert_eq!(MmapMode::AlwaysTryMmap, args.mmap); + + let args = parse_low_raw(["--no-mmap"]).unwrap(); + assert_eq!(MmapMode::Never, args.mmap); + + let args = parse_low_raw(["--mmap", "--no-mmap"]).unwrap(); + assert_eq!(MmapMode::Never, args.mmap); + + let args = parse_low_raw(["--no-mmap", "--mmap"]).unwrap(); + assert_eq!(MmapMode::AlwaysTryMmap, args.mmap); +} + +/// -U/--multiline +#[derive(Debug)] +struct Multiline; + +impl Flag for Multiline { + fn is_switch(&self) -> bool { + true + } + fn name_short(&self) -> Option { + Some(b'U') + } + fn name_long(&self) -> &'static str { + "multiline" + } + fn name_negated(&self) -> Option<&'static str> { + Some("no-multiline") + } + fn doc_category(&self) -> Category { + Category::Search + } + fn doc_short(&self) -> &'static str { + r"Enable searching across multiple lines." + } + fn doc_long(&self) -> &'static str { + r#" +This flag enables searching across multiple lines. +.sp +When multiline mode is enabled, ripgrep will lift the restriction that a +match cannot include a line terminator. For example, when multiline mode +is not enabled (the default), then the regex \fB\\p{any}\fP will match any +Unicode codepoint other than \fB\\n\fP. Similarly, the regex \fB\\n\fP is +explicitly forbidden, and if you try to use it, ripgrep will return an error. +However, when multiline mode is enabled, \fB\\p{any}\fP will match any Unicode +codepoint, including \fB\\n\fP, and regexes like \fB\\n\fP are permitted. +.sp +An important caveat is that multiline mode does not change the match semantics +of \fB.\fP. Namely, in most regex matchers, a \fB.\fP will by default match any +character other than \fB\\n\fP, and this is true in ripgrep as well. In order +to make \fB.\fP match \fB\\n\fP, you must enable the "dot all" flag inside the +regex. For example, both \fB(?s).\fP and \fB(?s:.)\fP have the same semantics, +where \fB.\fP will match any character, including \fB\\n\fP. Alternatively, the +\flag{multiline-dotall} flag may be passed to make the "dot all" behavior the +default. This flag only applies when multiline search is enabled. +.sp +There is no limit on the number of the lines that a single match can span. +.sp +\fBWARNING\fP: Because of how the underlying regex engine works, multiline +searches may be slower than normal line-oriented searches, and they may also +use more memory. In particular, when multiline mode is enabled, ripgrep +requires that each file it searches is laid out contiguously in memory (either +by reading it onto the heap or by memory-mapping it). Things that cannot be +memory-mapped (such as \fBstdin\fP) will be consumed until EOF before searching +can begin. In general, ripgrep will only do these things when necessary. +Specifically, if the \flag{multiline} flag is provided but the regex does +not contain patterns that would match \fB\\n\fP characters, then ripgrep +will automatically avoid reading each file into memory before searching it. +Nevertheless, if you only care about matches spanning at most one line, then it +is always better to disable multiline mode. +.sp +This overrides the \flag{stop-on-nonmatch} flag. +"# + } + + fn update(&self, v: FlagValue, args: &mut LowArgs) -> anyhow::Result<()> { + args.multiline = v.unwrap_switch(); + if args.multiline { + args.stop_on_nonmatch = false; + } + Ok(()) + } +} + +#[cfg(test)] +#[test] +fn test_multiline() { + let args = parse_low_raw(None::<&str>).unwrap(); + assert_eq!(false, args.multiline); + + let args = parse_low_raw(["--multiline"]).unwrap(); + assert_eq!(true, args.multiline); + + let args = parse_low_raw(["-U"]).unwrap(); + assert_eq!(true, args.multiline); + + let args = parse_low_raw(["-U", "--no-multiline"]).unwrap(); + assert_eq!(false, args.multiline); +} + +/// --multiline-dotall +#[derive(Debug)] +struct MultilineDotall; + +impl Flag for MultilineDotall { + fn is_switch(&self) -> bool { + true + } + fn name_long(&self) -> &'static str { + "multiline-dotall" + } + fn name_negated(&self) -> Option<&'static str> { + Some("no-multiline-dotall") + } + fn doc_category(&self) -> Category { + Category::Search + } + fn doc_short(&self) -> &'static str { + r"Make '.' match line terminators." + } + fn doc_long(&self) -> &'static str { + r#" +This flag enables "dot all" mode in all regex patterns. This causes \fB.\fP to +match line terminators when multiline searching is enabled. This flag has no +effect if multiline searching isn't enabled with the \flag{multiline} flag. +.sp +Normally, a \fB.\fP will match any character except line terminators. While +this behavior typically isn't relevant for line-oriented matching (since +matches can span at most one line), this can be useful when searching with the +\flag{multiline} flag. By default, multiline mode runs without "dot all" mode +enabled. +.sp +This flag is generally intended to be used in an alias or your ripgrep config +file if you prefer "dot all" semantics by default. Note that regardless of +whether this flag is used, "dot all" semantics can still be controlled via +inline flags in the regex pattern itself, e.g., \fB(?s:.)\fP always enables +"dot all" whereas \fB(?-s:.)\fP always disables "dot all". Moreover, you +can use character classes like \fB\\p{any}\fP to match any Unicode codepoint +regardless of whether "dot all" mode is enabled or not. +"# + } + + fn update(&self, v: FlagValue, args: &mut LowArgs) -> anyhow::Result<()> { + args.multiline_dotall = v.unwrap_switch(); + Ok(()) + } +} + +#[cfg(test)] +#[test] +fn test_multiline_dotall() { + let args = parse_low_raw(None::<&str>).unwrap(); + assert_eq!(false, args.multiline_dotall); + + let args = parse_low_raw(["--multiline-dotall"]).unwrap(); + assert_eq!(true, args.multiline_dotall); + + let args = parse_low_raw(["--multiline-dotall", "--no-multiline-dotall"]) + .unwrap(); + assert_eq!(false, args.multiline_dotall); +} + +/// --no-config +#[derive(Debug)] +struct NoConfig; + +impl Flag for NoConfig { + fn is_switch(&self) -> bool { + true + } + fn name_long(&self) -> &'static str { + "no-config" + } + fn doc_category(&self) -> Category { + Category::OtherBehaviors + } + fn doc_short(&self) -> &'static str { + r"Never read configuration files." + } + fn doc_long(&self) -> &'static str { + r" +When set, ripgrep will never read configuration files. When this flag is +present, ripgrep will not respect the \fBRIPGREP_CONFIG_PATH\fP environment +variable. +.sp +If ripgrep ever grows a feature to automatically read configuration files in +pre-defined locations, then this flag will also disable that behavior as well. +" + } + + fn update(&self, v: FlagValue, args: &mut LowArgs) -> anyhow::Result<()> { + assert!(v.unwrap_switch(), "--no-config has no negation"); + args.no_config = true; + Ok(()) + } +} + +#[cfg(test)] +#[test] +fn test_no_config() { + let args = parse_low_raw(None::<&str>).unwrap(); + assert_eq!(false, args.no_config); + + let args = parse_low_raw(["--no-config"]).unwrap(); + assert_eq!(true, args.no_config); +} + +/// --no-ignore +#[derive(Debug)] +struct NoIgnore; + +impl Flag for NoIgnore { + fn is_switch(&self) -> bool { + true + } + fn name_long(&self) -> &'static str { + "no-ignore" + } + fn name_negated(&self) -> Option<&'static str> { + Some("ignore") + } + fn doc_category(&self) -> Category { + Category::Filter + } + fn doc_short(&self) -> &'static str { + r"Don't use ignore files." + } + fn doc_long(&self) -> &'static str { + r" +When set, ignore files such as \fB.gitignore\fP, \fB.ignore\fP and +\fB.rgignore\fP will not be respected. This implies \flag{no-ignore-dot}, +\flag{no-ignore-exclude}, \flag{no-ignore-global}, \flag{no-ignore-parent} and +\flag{no-ignore-vcs}. +.sp +This does not imply \flag{no-ignore-files}, since \flag{ignore-file} is +specified explicitly as a command line argument. +.sp +When given only once, the \flag{unrestricted} flag is identical in +behavior to this flag and can be considered an alias. However, subsequent +\flag{unrestricted} flags have additional effects. +" + } + + fn update(&self, v: FlagValue, args: &mut LowArgs) -> anyhow::Result<()> { + let yes = v.unwrap_switch(); + args.no_ignore_dot = yes; + args.no_ignore_exclude = yes; + args.no_ignore_global = yes; + args.no_ignore_parent = yes; + args.no_ignore_vcs = yes; + Ok(()) + } +} + +#[cfg(test)] +#[test] +fn test_no_ignore() { + let args = parse_low_raw(None::<&str>).unwrap(); + assert_eq!(false, args.no_ignore_dot); + assert_eq!(false, args.no_ignore_exclude); + assert_eq!(false, args.no_ignore_global); + assert_eq!(false, args.no_ignore_parent); + assert_eq!(false, args.no_ignore_vcs); + + let args = parse_low_raw(["--no-ignore"]).unwrap(); + assert_eq!(true, args.no_ignore_dot); + assert_eq!(true, args.no_ignore_exclude); + assert_eq!(true, args.no_ignore_global); + assert_eq!(true, args.no_ignore_parent); + assert_eq!(true, args.no_ignore_vcs); + + let args = parse_low_raw(["--no-ignore", "--ignore"]).unwrap(); + assert_eq!(false, args.no_ignore_dot); + assert_eq!(false, args.no_ignore_exclude); + assert_eq!(false, args.no_ignore_global); + assert_eq!(false, args.no_ignore_parent); + assert_eq!(false, args.no_ignore_vcs); +} + +/// --no-ignore-dot +#[derive(Debug)] +struct NoIgnoreDot; + +impl Flag for NoIgnoreDot { + fn is_switch(&self) -> bool { + true + } + fn name_long(&self) -> &'static str { + "no-ignore-dot" + } + fn name_negated(&self) -> Option<&'static str> { + Some("ignore-dot") + } + fn doc_category(&self) -> Category { + Category::Filter + } + fn doc_short(&self) -> &'static str { + r"Don't use .ignore or .rgignore files." + } + fn doc_long(&self) -> &'static str { + r" +Don't respect filter rules from \fB.ignore\fP or \fB.rgignore\fP files. +.sp +This does not impact whether ripgrep will ignore files and directories whose +names begin with a dot. For that, see the \flag{hidden} flag. This flag also +does not impact whether filter rules from \fB.gitignore\fP files are respected. +" + } + + fn update(&self, v: FlagValue, args: &mut LowArgs) -> anyhow::Result<()> { + args.no_ignore_dot = v.unwrap_switch(); + Ok(()) + } +} + +#[cfg(test)] +#[test] +fn test_no_ignore_dot() { + let args = parse_low_raw(None::<&str>).unwrap(); + assert_eq!(false, args.no_ignore_dot); + + let args = parse_low_raw(["--no-ignore-dot"]).unwrap(); + assert_eq!(true, args.no_ignore_dot); + + let args = parse_low_raw(["--no-ignore-dot", "--ignore-dot"]).unwrap(); + assert_eq!(false, args.no_ignore_dot); +} + +/// --no-ignore-exclude +#[derive(Debug)] +struct NoIgnoreExclude; + +impl Flag for NoIgnoreExclude { + fn is_switch(&self) -> bool { + true + } + fn name_long(&self) -> &'static str { + "no-ignore-exclude" + } + fn name_negated(&self) -> Option<&'static str> { + Some("ignore-exclude") + } + fn doc_category(&self) -> Category { + Category::Filter + } + fn doc_short(&self) -> &'static str { + r"Don't use local exclusion files." + } + fn doc_long(&self) -> &'static str { + r" +Don't respect filter rules from files that are manually configured for the repository. +For example, this includes \fBgit\fP's \fB.git/info/exclude\fP. +" + } + + fn update(&self, v: FlagValue, args: &mut LowArgs) -> anyhow::Result<()> { + args.no_ignore_exclude = v.unwrap_switch(); + Ok(()) + } +} + +#[cfg(test)] +#[test] +fn test_no_ignore_exclude() { + let args = parse_low_raw(None::<&str>).unwrap(); + assert_eq!(false, args.no_ignore_exclude); + + let args = parse_low_raw(["--no-ignore-exclude"]).unwrap(); + assert_eq!(true, args.no_ignore_exclude); + + let args = + parse_low_raw(["--no-ignore-exclude", "--ignore-exclude"]).unwrap(); + assert_eq!(false, args.no_ignore_exclude); +} + +/// --no-ignore-files +#[derive(Debug)] +struct NoIgnoreFiles; + +impl Flag for NoIgnoreFiles { + fn is_switch(&self) -> bool { + true + } + fn name_long(&self) -> &'static str { + "no-ignore-files" + } + fn name_negated(&self) -> Option<&'static str> { + Some("ignore-files") + } + fn doc_category(&self) -> Category { + Category::Filter + } + fn doc_short(&self) -> &'static str { + r"Don't use --ignore-file arguments." + } + fn doc_long(&self) -> &'static str { + r" +When set, any \flag{ignore-file} flags, even ones that come after this flag, +are ignored. +" + } + + fn update(&self, v: FlagValue, args: &mut LowArgs) -> anyhow::Result<()> { + args.no_ignore_files = v.unwrap_switch(); + Ok(()) + } +} + +#[cfg(test)] +#[test] +fn test_no_ignore_files() { + let args = parse_low_raw(None::<&str>).unwrap(); + assert_eq!(false, args.no_ignore_files); + + let args = parse_low_raw(["--no-ignore-files"]).unwrap(); + assert_eq!(true, args.no_ignore_files); + + let args = parse_low_raw(["--no-ignore-files", "--ignore-files"]).unwrap(); + assert_eq!(false, args.no_ignore_files); +} + +/// --no-ignore-global +#[derive(Debug)] +struct NoIgnoreGlobal; + +impl Flag for NoIgnoreGlobal { + fn is_switch(&self) -> bool { + true + } + fn name_long(&self) -> &'static str { + "no-ignore-global" + } + fn name_negated(&self) -> Option<&'static str> { + Some("ignore-global") + } + fn doc_category(&self) -> Category { + Category::Filter + } + fn doc_short(&self) -> &'static str { + r"Don't use global ignore files." + } + fn doc_long(&self) -> &'static str { + r#" +Don't respect filter rules from ignore files that come from "global" sources +such as \fBgit\fP's \fBcore.excludesFile\fP configuration option (which +defaults to \fB$HOME/.config/git/ignore\fP). +"# + } + + fn update(&self, v: FlagValue, args: &mut LowArgs) -> anyhow::Result<()> { + args.no_ignore_global = v.unwrap_switch(); + Ok(()) + } +} + +#[cfg(test)] +#[test] +fn test_no_ignore_global() { + let args = parse_low_raw(None::<&str>).unwrap(); + assert_eq!(false, args.no_ignore_global); + + let args = parse_low_raw(["--no-ignore-global"]).unwrap(); + assert_eq!(true, args.no_ignore_global); + + let args = + parse_low_raw(["--no-ignore-global", "--ignore-global"]).unwrap(); + assert_eq!(false, args.no_ignore_global); +} + +/// --no-ignore-messages +#[derive(Debug)] +struct NoIgnoreMessages; + +impl Flag for NoIgnoreMessages { + fn is_switch(&self) -> bool { + true + } + fn name_long(&self) -> &'static str { + "no-ignore-messages" + } + fn name_negated(&self) -> Option<&'static str> { + Some("ignore-messages") + } + fn doc_category(&self) -> Category { + Category::Logging + } + fn doc_short(&self) -> &'static str { + r"Suppress gitignore parse error messages." + } + fn doc_long(&self) -> &'static str { + r" +When this flag is enabled, all error messages related to parsing ignore files +are suppressed. By default, error messages are printed to stderr. In cases +where these errors are expected, this flag can be used to avoid seeing the +noise produced by the messages. +" + } + + fn update(&self, v: FlagValue, args: &mut LowArgs) -> anyhow::Result<()> { + args.no_ignore_messages = v.unwrap_switch(); + Ok(()) + } +} + +#[cfg(test)] +#[test] +fn test_no_ignore_messages() { + let args = parse_low_raw(None::<&str>).unwrap(); + assert_eq!(false, args.no_ignore_messages); + + let args = parse_low_raw(["--no-ignore-messages"]).unwrap(); + assert_eq!(true, args.no_ignore_messages); + + let args = + parse_low_raw(["--no-ignore-messages", "--ignore-messages"]).unwrap(); + assert_eq!(false, args.no_ignore_messages); +} + +/// --no-ignore-parent +#[derive(Debug)] +struct NoIgnoreParent; + +impl Flag for NoIgnoreParent { + fn is_switch(&self) -> bool { + true + } + fn name_long(&self) -> &'static str { + "no-ignore-parent" + } + fn name_negated(&self) -> Option<&'static str> { + Some("ignore-parent") + } + fn doc_category(&self) -> Category { + Category::Filter + } + fn doc_short(&self) -> &'static str { + r"Don't use ignore files in parent directories." + } + fn doc_long(&self) -> &'static str { + r" +When this flag is set, filter rules from ignore files found in parent +directories are not respected. By default, ripgrep will ascend the parent +directories of the current working directory to look for any applicable ignore +files that should be applied. In some cases this may not be desirable. +" + } + + fn update(&self, v: FlagValue, args: &mut LowArgs) -> anyhow::Result<()> { + args.no_ignore_parent = v.unwrap_switch(); + Ok(()) + } +} + +#[cfg(test)] +#[test] +fn test_no_ignore_parent() { + let args = parse_low_raw(None::<&str>).unwrap(); + assert_eq!(false, args.no_ignore_parent); + + let args = parse_low_raw(["--no-ignore-parent"]).unwrap(); + assert_eq!(true, args.no_ignore_parent); + + let args = + parse_low_raw(["--no-ignore-parent", "--ignore-parent"]).unwrap(); + assert_eq!(false, args.no_ignore_parent); +} + +/// --no-ignore-vcs +#[derive(Debug)] +struct NoIgnoreVcs; + +impl Flag for NoIgnoreVcs { + fn is_switch(&self) -> bool { + true + } + fn name_long(&self) -> &'static str { + "no-ignore-vcs" + } + fn name_negated(&self) -> Option<&'static str> { + Some("ignore-vcs") + } + fn doc_category(&self) -> Category { + Category::Filter + } + fn doc_short(&self) -> &'static str { + r"Don't use ignore files from source control." + } + fn doc_long(&self) -> &'static str { + r" +When given, filter rules from source control ignore files (e.g., +\fB.gitignore\fP) are not respected. By default, ripgrep respects \fBgit\fP's +ignore rules for automatic filtering. In some cases, it may not be desirable +to respect the source control's ignore rules and instead only respect rules in +\fB.ignore\fP or \fB.rgignore\fP. +.sp +Note that this flag does not directly affect the filtering of source control +files or folders that start with a dot (\fB.\fP), like \fB.git\fP. These are +affected by \flag{hidden} and its related flags instead. +.sp +This flag implies \flag{no-ignore-parent} for source control ignore files as +well. +" + } + + fn update(&self, v: FlagValue, args: &mut LowArgs) -> anyhow::Result<()> { + args.no_ignore_vcs = v.unwrap_switch(); + Ok(()) + } +} + +#[cfg(test)] +#[test] +fn test_no_ignore_vcs() { + let args = parse_low_raw(None::<&str>).unwrap(); + assert_eq!(false, args.no_ignore_vcs); + + let args = parse_low_raw(["--no-ignore-vcs"]).unwrap(); + assert_eq!(true, args.no_ignore_vcs); + + let args = parse_low_raw(["--no-ignore-vcs", "--ignore-vcs"]).unwrap(); + assert_eq!(false, args.no_ignore_vcs); +} + +/// --no-messages +#[derive(Debug)] +struct NoMessages; + +impl Flag for NoMessages { + fn is_switch(&self) -> bool { + true + } + fn name_long(&self) -> &'static str { + "no-messages" + } + fn name_negated(&self) -> Option<&'static str> { + Some("messages") + } + fn doc_category(&self) -> Category { + Category::Logging + } + fn doc_short(&self) -> &'static str { + r"Suppress some error messages." + } + fn doc_long(&self) -> &'static str { + r" +This flag suppresses some error messages. Specifically, messages related to +the failed opening and reading of files. Error messages related to the syntax +of the pattern are still shown. +" + } + + fn update(&self, v: FlagValue, args: &mut LowArgs) -> anyhow::Result<()> { + args.no_messages = v.unwrap_switch(); + Ok(()) + } +} + +#[cfg(test)] +#[test] +fn test_no_messages() { + let args = parse_low_raw(None::<&str>).unwrap(); + assert_eq!(false, args.no_messages); + + let args = parse_low_raw(["--no-messages"]).unwrap(); + assert_eq!(true, args.no_messages); + + let args = parse_low_raw(["--no-messages", "--messages"]).unwrap(); + assert_eq!(false, args.no_messages); +} + +/// --no-pcre2-unicode +#[derive(Debug)] +struct NoPcre2Unicode; + +impl Flag for NoPcre2Unicode { + fn is_switch(&self) -> bool { + true + } + fn name_long(&self) -> &'static str { + "no-pcre2-unicode" + } + fn name_negated(&self) -> Option<&'static str> { + Some("pcre2-unicode") + } + fn doc_category(&self) -> Category { + Category::Search + } + fn doc_short(&self) -> &'static str { + r"(DEPRECATED) Disable Unicode mode for PCRE2." + } + fn doc_long(&self) -> &'static str { + r" +DEPRECATED. Use \flag{no-unicode} instead. +.sp +Note that Unicode mode is enabled by default. +" + } + + fn update(&self, v: FlagValue, args: &mut LowArgs) -> anyhow::Result<()> { + args.no_unicode = v.unwrap_switch(); + Ok(()) + } +} + +#[cfg(test)] +#[test] +fn test_no_pcre2_unicode() { + let args = parse_low_raw(None::<&str>).unwrap(); + assert_eq!(false, args.no_unicode); + + let args = parse_low_raw(["--no-pcre2-unicode"]).unwrap(); + assert_eq!(true, args.no_unicode); + + let args = + parse_low_raw(["--no-pcre2-unicode", "--pcre2-unicode"]).unwrap(); + assert_eq!(false, args.no_unicode); +} + +/// --no-require-git +#[derive(Debug)] +struct NoRequireGit; + +impl Flag for NoRequireGit { + fn is_switch(&self) -> bool { + true + } + fn name_long(&self) -> &'static str { + "no-require-git" + } + fn name_negated(&self) -> Option<&'static str> { + Some("require-git") + } + fn doc_category(&self) -> Category { + Category::Filter + } + fn doc_short(&self) -> &'static str { + r"Use .gitignore outside of git repositories." + } + fn doc_long(&self) -> &'static str { + r" +When this flag is given, source control ignore files such as \fB.gitignore\fP +are respected even if no \fBgit\fP repository is present. +.sp +By default, ripgrep will only respect filter rules from source control ignore +files when ripgrep detects that the search is executed inside a source control +repository. For example, when a \fB.git\fP directory is observed. +.sp +This flag relaxes the default restriction. For example, it might be useful when +the contents of a \fBgit\fP repository are stored or copied somewhere, but +where the repository state is absent. +" + } + + fn update(&self, v: FlagValue, args: &mut LowArgs) -> anyhow::Result<()> { + args.no_require_git = v.unwrap_switch(); + Ok(()) + } +} + +#[cfg(test)] +#[test] +fn test_no_require_git() { + let args = parse_low_raw(None::<&str>).unwrap(); + assert_eq!(false, args.no_require_git); + + let args = parse_low_raw(["--no-require-git"]).unwrap(); + assert_eq!(true, args.no_require_git); + + let args = parse_low_raw(["--no-require-git", "--require-git"]).unwrap(); + assert_eq!(false, args.no_require_git); +} + +/// --no-unicode +#[derive(Debug)] +struct NoUnicode; + +impl Flag for NoUnicode { + fn is_switch(&self) -> bool { + true + } + fn name_long(&self) -> &'static str { + "no-unicode" + } + fn name_negated(&self) -> Option<&'static str> { + Some("unicode") + } + fn doc_category(&self) -> Category { + Category::Search + } + fn doc_short(&self) -> &'static str { + r"Disable Unicode mode." + } + fn doc_long(&self) -> &'static str { + r#" +This flag disables Unicode mode for all patterns given to ripgrep. +.sp +By default, ripgrep will enable "Unicode mode" in all of its regexes. This has +a number of consequences: +.sp +.IP \(bu 3n +\fB.\fP will only match valid UTF-8 encoded Unicode scalar values. +.sp +.IP \(bu 3n +Classes like \fB\\w\fP, \fB\\s\fP, \fB\\d\fP are all Unicode aware and much +bigger than their ASCII only versions. +.sp +.IP \(bu 3n +Case insensitive matching will use Unicode case folding. +.sp +.IP \(bu 3n +A large array of classes like \fB\\p{Emoji}\fP are available. (Although the +specific set of classes available varies based on the regex engine. In general, +the default regex engine has more classes available to it.) +.sp +.IP \(bu 3n +Word boundaries (\fB\\b\fP and \fB\\B\fP) use the Unicode definition of a word +character. +.PP +In some cases it can be desirable to turn these things off. This flag will do +exactly that. For example, Unicode mode can sometimes have a negative impact +on performance, especially when things like \fB\\w\fP are used frequently +(including via bounded repetitions like \fB\\w{100}\fP) when only their ASCII +interpretation is needed. +"# + } + + fn update(&self, v: FlagValue, args: &mut LowArgs) -> anyhow::Result<()> { + args.no_unicode = v.unwrap_switch(); + Ok(()) + } +} + +#[cfg(test)] +#[test] +fn test_no_unicode() { + let args = parse_low_raw(None::<&str>).unwrap(); + assert_eq!(false, args.no_unicode); + + let args = parse_low_raw(["--no-unicode"]).unwrap(); + assert_eq!(true, args.no_unicode); + + let args = parse_low_raw(["--no-unicode", "--unicode"]).unwrap(); + assert_eq!(false, args.no_unicode); + + let args = parse_low_raw(["--no-unicode", "--pcre2-unicode"]).unwrap(); + assert_eq!(false, args.no_unicode); + + let args = parse_low_raw(["--no-pcre2-unicode", "--unicode"]).unwrap(); + assert_eq!(false, args.no_unicode); +} + +/// -0/--null +#[derive(Debug)] +struct Null; + +impl Flag for Null { + fn is_switch(&self) -> bool { + true + } + fn name_short(&self) -> Option { + Some(b'0') + } + fn name_long(&self) -> &'static str { + "null" + } + fn doc_category(&self) -> Category { + Category::Output + } + fn doc_short(&self) -> &'static str { + r"Print a NUL byte after file paths." + } + fn doc_long(&self) -> &'static str { + r" +Whenever a file path is printed, follow it with a \fBNUL\fP byte. This includes +printing file paths before matches, and when printing a list of matching files +such as with \flag{count}, \flag{files-with-matches} and \flag{files}. This +option is useful for use with \fBxargs\fP. +" + } + + fn update(&self, v: FlagValue, args: &mut LowArgs) -> anyhow::Result<()> { + assert!(v.unwrap_switch(), "--null has no negation"); + args.null = true; + Ok(()) + } +} + +#[cfg(test)] +#[test] +fn test_null() { + let args = parse_low_raw(None::<&str>).unwrap(); + assert_eq!(false, args.null); + + let args = parse_low_raw(["--null"]).unwrap(); + assert_eq!(true, args.null); + + let args = parse_low_raw(["-0"]).unwrap(); + assert_eq!(true, args.null); +} + +/// --null-data +#[derive(Debug)] +struct NullData; + +impl Flag for NullData { + fn is_switch(&self) -> bool { + true + } + fn name_long(&self) -> &'static str { + "null-data" + } + fn doc_category(&self) -> Category { + Category::Search + } + fn doc_short(&self) -> &'static str { + r"Use NUL as a line terminator." + } + fn doc_long(&self) -> &'static str { + r" +Enabling this flag causes ripgrep to use \fBNUL\fP as a line terminator instead +of the default of \fP\\n\fP. +.sp +This is useful when searching large binary files that would otherwise have +very long lines if \fB\\n\fP were used as the line terminator. In particular, +ripgrep requires that, at a minimum, each line must fit into memory. Using +\fBNUL\fP instead can be a useful stopgap to keep memory requirements low and +avoid OOM (out of memory) conditions. +.sp +This is also useful for processing NUL delimited data, such as that emitted +when using ripgrep's \flag{null} flag or \fBfind\fP's \fB\-\-print0\fP flag. +.sp +Using this flag implies \flag{text}. It also overrides \flag{crlf}. +" + } + + fn update(&self, v: FlagValue, args: &mut LowArgs) -> anyhow::Result<()> { + assert!(v.unwrap_switch(), "--null-data has no negation"); + args.crlf = false; + args.null_data = true; + Ok(()) + } +} + +#[cfg(test)] +#[test] +fn test_null_data() { + let args = parse_low_raw(None::<&str>).unwrap(); + assert_eq!(false, args.null_data); + + let args = parse_low_raw(["--null-data"]).unwrap(); + assert_eq!(true, args.null_data); + + let args = parse_low_raw(["--null-data", "--crlf"]).unwrap(); + assert_eq!(false, args.null_data); + assert_eq!(true, args.crlf); + + let args = parse_low_raw(["--crlf", "--null-data"]).unwrap(); + assert_eq!(true, args.null_data); + assert_eq!(false, args.crlf); + + let args = parse_low_raw(["--null-data", "--no-crlf"]).unwrap(); + assert_eq!(true, args.null_data); + assert_eq!(false, args.crlf); +} + +/// --one-file-system +#[derive(Debug)] +struct OneFileSystem; + +impl Flag for OneFileSystem { + fn is_switch(&self) -> bool { + true + } + fn name_long(&self) -> &'static str { + "one-file-system" + } + fn name_negated(&self) -> Option<&'static str> { + Some("no-one-file-system") + } + fn doc_category(&self) -> Category { + Category::Filter + } + fn doc_short(&self) -> &'static str { + r"Skip directories on other file systems." + } + fn doc_long(&self) -> &'static str { + r" +When enabled, ripgrep will not cross file system boundaries relative to where +the search started from. +.sp +Note that this applies to each path argument given to ripgrep. For example, in +the command +.sp +.EX + rg \-\-one\-file\-system /foo/bar /quux/baz +.EE +.sp +ripgrep will search both \fI/foo/bar\fP and \fI/quux/baz\fP even if they are +on different file systems, but will not cross a file system boundary when +traversing each path's directory tree. +.sp +This is similar to \fBfind\fP's \fB\-xdev\fP or \fB\-mount\fP flag. +" + } + + fn update(&self, v: FlagValue, args: &mut LowArgs) -> anyhow::Result<()> { + args.one_file_system = v.unwrap_switch(); + Ok(()) + } +} + +#[cfg(test)] +#[test] +fn test_one_file_system() { + let args = parse_low_raw(None::<&str>).unwrap(); + assert_eq!(false, args.one_file_system); + + let args = parse_low_raw(["--one-file-system"]).unwrap(); + assert_eq!(true, args.one_file_system); + + let args = + parse_low_raw(["--one-file-system", "--no-one-file-system"]).unwrap(); + assert_eq!(false, args.one_file_system); +} + +/// -o/--only-matching +#[derive(Debug)] +struct OnlyMatching; + +impl Flag for OnlyMatching { + fn is_switch(&self) -> bool { + true + } + fn name_short(&self) -> Option { + Some(b'o') + } + fn name_long(&self) -> &'static str { + "only-matching" + } + fn doc_category(&self) -> Category { + Category::Output + } + fn doc_short(&self) -> &'static str { + r"Print only matched parts of a line." + } + fn doc_long(&self) -> &'static str { + r" +Print only the matched (non-empty) parts of a matching line, with each such +part on a separate output line. +" + } + + fn update(&self, v: FlagValue, args: &mut LowArgs) -> anyhow::Result<()> { + assert!(v.unwrap_switch(), "--only-matching does not have a negation"); + args.only_matching = true; + Ok(()) + } +} + +#[cfg(test)] +#[test] +fn test_only_matching() { + let args = parse_low_raw(None::<&str>).unwrap(); + assert_eq!(false, args.only_matching); + + let args = parse_low_raw(["--only-matching"]).unwrap(); + assert_eq!(true, args.only_matching); + + let args = parse_low_raw(["-o"]).unwrap(); + assert_eq!(true, args.only_matching); +} + +/// --path-separator +#[derive(Debug)] +struct PathSeparator; + +impl Flag for PathSeparator { + fn is_switch(&self) -> bool { + false + } + fn name_long(&self) -> &'static str { + "path-separator" + } + fn doc_variable(&self) -> Option<&'static str> { + Some("SEPARATOR") + } + fn doc_category(&self) -> Category { + Category::Output + } + fn doc_short(&self) -> &'static str { + r"Set the path separator for printing paths." + } + fn doc_long(&self) -> &'static str { + r" +Set the path separator to use when printing file paths. This defaults to your +platform's path separator, which is \fB/\fP on Unix and \fB\\\fP on Windows. +This flag is intended for overriding the default when the environment demands +it (e.g., cygwin). A path separator is limited to a single byte. +.sp +Setting this flag to an empty string reverts it to its default behavior. That +is, the path separator is automatically chosen based on the environment. +" + } + + fn update(&self, v: FlagValue, args: &mut LowArgs) -> anyhow::Result<()> { + let s = convert::string(v.unwrap_value())?; + let raw = Vec::unescape_bytes(&s); + args.path_separator = if raw.is_empty() { + None + } else if raw.len() == 1 { + Some(raw[0]) + } else { + anyhow::bail!( + "A path separator must be exactly one byte, but \ + the given separator is {len} bytes: {sep}\n\ + In some shells on Windows '/' is automatically \ + expanded. Use '//' instead.", + len = raw.len(), + sep = s, + ) + }; + Ok(()) + } +} + +#[cfg(test)] +#[test] +fn test_path_separator() { + let args = parse_low_raw(None::<&str>).unwrap(); + assert_eq!(None, args.path_separator); + + let args = parse_low_raw(["--path-separator", "/"]).unwrap(); + assert_eq!(Some(b'/'), args.path_separator); + + let args = parse_low_raw(["--path-separator", r"\"]).unwrap(); + assert_eq!(Some(b'\\'), args.path_separator); + + let args = parse_low_raw(["--path-separator", r"\x00"]).unwrap(); + assert_eq!(Some(0), args.path_separator); + + let args = parse_low_raw(["--path-separator", r"\0"]).unwrap(); + assert_eq!(Some(0), args.path_separator); + + let args = parse_low_raw(["--path-separator", "\x00"]).unwrap(); + assert_eq!(Some(0), args.path_separator); + + let args = parse_low_raw(["--path-separator", "\0"]).unwrap(); + assert_eq!(Some(0), args.path_separator); + + let args = + parse_low_raw(["--path-separator", r"\x00", "--path-separator=/"]) + .unwrap(); + assert_eq!(Some(b'/'), args.path_separator); + + let result = parse_low_raw(["--path-separator", "foo"]); + assert!(result.is_err(), "{result:?}"); + + let result = parse_low_raw(["--path-separator", r"\\x00"]); + assert!(result.is_err(), "{result:?}"); +} + +/// --passthru +#[derive(Debug)] +struct Passthru; + +impl Flag for Passthru { + fn is_switch(&self) -> bool { + true + } + fn name_long(&self) -> &'static str { + "passthru" + } + fn aliases(&self) -> &'static [&'static str] { + &["passthrough"] + } + fn doc_category(&self) -> Category { + Category::Output + } + fn doc_short(&self) -> &'static str { + r"Print both matching and non-matching lines." + } + fn doc_long(&self) -> &'static str { + r#" +Print both matching and non-matching lines. +.sp +Another way to achieve a similar effect is by modifying your pattern to match +the empty string. For example, if you are searching using \fBrg\fP \fIfoo\fP, +then using \fBrg\fP \fB'^|\fP\fIfoo\fP\fB'\fP instead will emit every line in +every file searched, but only occurrences of \fIfoo\fP will be highlighted. +This flag enables the same behavior without needing to modify the pattern. +.sp +An alternative spelling for this flag is \fB\-\-passthrough\fP. +.sp +This overrides the \flag{context}, \flag{after-context} and +\flag{before-context} flags. +"# + } + + fn update(&self, v: FlagValue, args: &mut LowArgs) -> anyhow::Result<()> { + assert!(v.unwrap_switch(), "--passthru has no negation"); + args.context = ContextMode::Passthru; + Ok(()) + } +} + +#[cfg(test)] +#[test] +fn test_passthru() { + let args = parse_low_raw(None::<&str>).unwrap(); + assert_eq!(ContextMode::default(), args.context); + + let args = parse_low_raw(["--passthru"]).unwrap(); + assert_eq!(ContextMode::Passthru, args.context); + + let args = parse_low_raw(["--passthrough"]).unwrap(); + assert_eq!(ContextMode::Passthru, args.context); +} + +/// -P/--pcre2 +#[derive(Debug)] +struct PCRE2; + +impl Flag for PCRE2 { + fn is_switch(&self) -> bool { + true + } + fn name_short(&self) -> Option { + Some(b'P') + } + fn name_long(&self) -> &'static str { + "pcre2" + } + fn name_negated(&self) -> Option<&'static str> { + Some("no-pcre2") + } + fn doc_category(&self) -> Category { + Category::Search + } + fn doc_short(&self) -> &'static str { + r"Enable PCRE2 matching." + } + fn doc_long(&self) -> &'static str { + r" +When this flag is present, ripgrep will use the PCRE2 regex engine instead of +its default regex engine. +.sp +This is generally useful when you want to use features such as look-around +or backreferences. +.sp +Using this flag is the same as passing \fB\-\-engine=pcre2\fP. Users may +instead elect to use \fB\-\-engine=auto\fP to ask ripgrep to automatically +select the right regex engine based on the patterns given. This flag and the +\flag{engine} flag override one another. +.sp +Note that PCRE2 is an optional ripgrep feature. If PCRE2 wasn't included in +your build of ripgrep, then using this flag will result in ripgrep printing +an error message and exiting. PCRE2 may also have worse user experience in +some cases, since it has fewer introspection APIs than ripgrep's default +regex engine. For example, if you use a \fB\\n\fP in a PCRE2 regex without +the \flag{multiline} flag, then ripgrep will silently fail to match anything +instead of reporting an error immediately (like it does with the default regex +engine). +" + } + + fn update(&self, v: FlagValue, args: &mut LowArgs) -> anyhow::Result<()> { + args.engine = if v.unwrap_switch() { + EngineChoice::PCRE2 + } else { + EngineChoice::Default + }; + Ok(()) + } +} + +#[cfg(test)] +#[test] +fn test_pcre2() { + let args = parse_low_raw(None::<&str>).unwrap(); + assert_eq!(EngineChoice::Default, args.engine); + + let args = parse_low_raw(["--pcre2"]).unwrap(); + assert_eq!(EngineChoice::PCRE2, args.engine); + + let args = parse_low_raw(["-P"]).unwrap(); + assert_eq!(EngineChoice::PCRE2, args.engine); + + let args = parse_low_raw(["-P", "--no-pcre2"]).unwrap(); + assert_eq!(EngineChoice::Default, args.engine); + + let args = parse_low_raw(["--engine=auto", "-P", "--no-pcre2"]).unwrap(); + assert_eq!(EngineChoice::Default, args.engine); + + let args = parse_low_raw(["-P", "--engine=auto"]).unwrap(); + assert_eq!(EngineChoice::Auto, args.engine); +} + +/// --pcre2-version +#[derive(Debug)] +struct PCRE2Version; + +impl Flag for PCRE2Version { + fn is_switch(&self) -> bool { + true + } + fn name_long(&self) -> &'static str { + "pcre2-version" + } + fn doc_category(&self) -> Category { + Category::OtherBehaviors + } + fn doc_short(&self) -> &'static str { + r"Print the version of PCRE2 that ripgrep uses." + } + fn doc_long(&self) -> &'static str { + r" +When this flag is present, ripgrep will print the version of PCRE2 in use, +along with other information, and then exit. If PCRE2 is not available, then +ripgrep will print an error message and exit with an error code. +" + } + + fn update(&self, v: FlagValue, args: &mut LowArgs) -> anyhow::Result<()> { + assert!(v.unwrap_switch(), "--pcre2-version has no negation"); + args.special = Some(SpecialMode::VersionPCRE2); + Ok(()) + } +} + +#[cfg(test)] +#[test] +fn test_pcre2_version() { + let args = parse_low_raw(None::<&str>).unwrap(); + assert_eq!(None, args.special); + + let args = parse_low_raw(["--pcre2-version"]).unwrap(); + assert_eq!(Some(SpecialMode::VersionPCRE2), args.special); +} + +/// --pre +#[derive(Debug)] +struct Pre; + +impl Flag for Pre { + fn is_switch(&self) -> bool { + false + } + fn name_long(&self) -> &'static str { + "pre" + } + fn name_negated(&self) -> Option<&'static str> { + Some("no-pre") + } + fn doc_variable(&self) -> Option<&'static str> { + Some("COMMAND") + } + fn doc_category(&self) -> Category { + Category::Input + } + fn doc_short(&self) -> &'static str { + r"Search output of COMMAND for each PATH." + } + fn doc_long(&self) -> &'static str { + r#" +For each input \fIPATH\fP, this flag causes ripgrep to search the standard +output of \fICOMMAND\fP \fIPATH\fP instead of the contents of \fIPATH\fP. +This option expects the \fICOMMAND\fP program to either be a path or to be +available in your \fBPATH\fP. Either an empty string \fICOMMAND\fP or the +\fB\-\-no\-pre\fP flag will disable this behavior. +.sp +.TP 12 +\fBWARNING\fP +When this flag is set, ripgrep will unconditionally spawn a process for every +file that is searched. Therefore, this can incur an unnecessarily large +performance penalty if you don't otherwise need the flexibility offered by this +flag. One possible mitigation to this is to use the \flag{pre-glob} flag to +limit which files a preprocessor is run with. +.PP +A preprocessor is not run when ripgrep is searching stdin. +.sp +When searching over sets of files that may require one of several +preprocessors, \fICOMMAND\fP should be a wrapper program which first classifies +\fIPATH\fP based on magic numbers/content or based on the \fIPATH\fP name and +then dispatches to an appropriate preprocessor. Each \fICOMMAND\fP also has its +standard input connected to \fIPATH\fP for convenience. +.sp +For example, a shell script for \fICOMMAND\fP might look like: +.sp +.EX + case "$1" in + *.pdf) + exec pdftotext "$1" - + ;; + *) + case $(file "$1") in + *Zstandard*) + exec pzstd -cdq + ;; + *) + exec cat + ;; + esac + ;; + esac +.EE +.sp +The above script uses \fBpdftotext\fP to convert a PDF file to plain text. For +all other files, the script uses the \fBfile\fP utility to sniff the type of +the file based on its contents. If it is a compressed file in the Zstandard +format, then \fBpzstd\fP is used to decompress the contents to stdout. +.sp +This overrides the \flag{search-zip} flag. +"# + } + fn completion_type(&self) -> CompletionType { + CompletionType::Executable + } + + fn update(&self, v: FlagValue, args: &mut LowArgs) -> anyhow::Result<()> { + let path = match v { + FlagValue::Value(v) => PathBuf::from(v), + FlagValue::Switch(yes) => { + assert!(!yes, "there is no affirmative switch for --pre"); + args.pre = None; + return Ok(()); + } + }; + args.pre = if path.as_os_str().is_empty() { None } else { Some(path) }; + if args.pre.is_some() { + args.search_zip = false; + } + Ok(()) + } +} + +#[cfg(test)] +#[test] +fn test_pre() { + let args = parse_low_raw(None::<&str>).unwrap(); + assert_eq!(None, args.pre); + + let args = parse_low_raw(["--pre", "foo/bar"]).unwrap(); + assert_eq!(Some(PathBuf::from("foo/bar")), args.pre); + + let args = parse_low_raw(["--pre", ""]).unwrap(); + assert_eq!(None, args.pre); + + let args = parse_low_raw(["--pre", "foo/bar", "--pre", ""]).unwrap(); + assert_eq!(None, args.pre); + + let args = parse_low_raw(["--pre", "foo/bar", "--pre="]).unwrap(); + assert_eq!(None, args.pre); + + let args = parse_low_raw(["--pre", "foo/bar", "--no-pre"]).unwrap(); + assert_eq!(None, args.pre); +} + +/// --pre-glob +#[derive(Debug)] +struct PreGlob; + +impl Flag for PreGlob { + fn is_switch(&self) -> bool { + false + } + fn name_long(&self) -> &'static str { + "pre-glob" + } + fn doc_variable(&self) -> Option<&'static str> { + Some("GLOB") + } + fn doc_category(&self) -> Category { + Category::Input + } + fn doc_short(&self) -> &'static str { + r"Include or exclude files from a preprocessor." + } + fn doc_long(&self) -> &'static str { + r#" +This flag works in conjunction with the \flag{pre} flag. Namely, when one or +more \flag{pre-glob} flags are given, then only files that match the given set +of globs will be handed to the command specified by the \flag{pre} flag. Any +non-matching files will be searched without using the preprocessor command. +.sp +This flag is useful when searching many files with the \flag{pre} flag. +Namely, it provides the ability to avoid process overhead for files that +don't need preprocessing. For example, given the following shell script, +\fIpre-pdftotext\fP: +.sp +.EX + #!/bin/sh + pdftotext "$1" - +.EE +.sp +then it is possible to use \fB\-\-pre\fP \fIpre-pdftotext\fP +\fB\-\-pre\-glob\fP '\fI*.pdf\fP' to make it so ripgrep only executes +the \fIpre-pdftotext\fP command on files with a \fI.pdf\fP extension. +.sp +Multiple \flag{pre-glob} flags may be used. Globbing rules match +\fBgitignore\fP globs. Precede a glob with a \fB!\fP to exclude it. +.sp +This flag has no effect if the \flag{pre} flag is not used. +"# + } + + fn update(&self, v: FlagValue, args: &mut LowArgs) -> anyhow::Result<()> { + let glob = convert::string(v.unwrap_value())?; + args.pre_glob.push(glob); + Ok(()) + } +} + +#[cfg(test)] +#[test] +fn test_pre_glob() { + let args = parse_low_raw(None::<&str>).unwrap(); + assert_eq!(Vec::::new(), args.pre_glob); + + let args = parse_low_raw(["--pre-glob", "*.pdf"]).unwrap(); + assert_eq!(vec!["*.pdf".to_string()], args.pre_glob); + + let args = + parse_low_raw(["--pre-glob", "*.pdf", "--pre-glob=foo"]).unwrap(); + assert_eq!(vec!["*.pdf".to_string(), "foo".to_string()], args.pre_glob); +} + +/// -p/--pretty +#[derive(Debug)] +struct Pretty; + +impl Flag for Pretty { + fn is_switch(&self) -> bool { + true + } + fn name_short(&self) -> Option { + Some(b'p') + } + fn name_long(&self) -> &'static str { + "pretty" + } + fn doc_category(&self) -> Category { + Category::Output + } + fn doc_short(&self) -> &'static str { + r"Alias for colors, headings and line numbers." + } + fn doc_long(&self) -> &'static str { + r" +This is a convenience alias for \fB\-\-color=always \-\-heading +\-\-line\-number\fP. This flag is useful when you still want pretty output even +if you're piping ripgrep to another program or file. For example: \fBrg -p +\fP\fIfoo\fP \fB| less -R\fP. +" + } + + fn update(&self, v: FlagValue, args: &mut LowArgs) -> anyhow::Result<()> { + assert!(v.unwrap_switch(), "--pretty has no negation"); + args.color = ColorChoice::Always; + args.heading = Some(true); + args.line_number = Some(true); + Ok(()) + } +} + +#[cfg(test)] +#[test] +fn test_pretty() { + let args = parse_low_raw(None::<&str>).unwrap(); + assert_eq!(ColorChoice::Auto, args.color); + assert_eq!(None, args.heading); + assert_eq!(None, args.line_number); + + let args = parse_low_raw(["--pretty"]).unwrap(); + assert_eq!(ColorChoice::Always, args.color); + assert_eq!(Some(true), args.heading); + assert_eq!(Some(true), args.line_number); + + let args = parse_low_raw(["-p"]).unwrap(); + assert_eq!(ColorChoice::Always, args.color); + assert_eq!(Some(true), args.heading); + assert_eq!(Some(true), args.line_number); +} + +/// -q/--quiet +#[derive(Debug)] +struct Quiet; + +impl Flag for Quiet { + fn is_switch(&self) -> bool { + true + } + fn name_short(&self) -> Option { + Some(b'q') + } + fn name_long(&self) -> &'static str { + "quiet" + } + fn doc_category(&self) -> Category { + Category::Output + } + fn doc_short(&self) -> &'static str { + r"Do not print anything to stdout." + } + fn doc_long(&self) -> &'static str { + r" +Do not print anything to stdout. If a match is found in a file, then ripgrep +will stop searching. This is useful when ripgrep is used only for its exit code +(which will be an error code if no matches are found). +.sp +When \flag{files} is used, ripgrep will stop finding files after finding the +first file that does not match any ignore rules. +" + } + + fn update(&self, v: FlagValue, args: &mut LowArgs) -> anyhow::Result<()> { + assert!(v.unwrap_switch(), "--quiet has no negation"); + args.quiet = true; + Ok(()) + } +} + +#[cfg(test)] +#[test] +fn test_quiet() { + let args = parse_low_raw(None::<&str>).unwrap(); + assert_eq!(false, args.quiet); + + let args = parse_low_raw(["--quiet"]).unwrap(); + assert_eq!(true, args.quiet); + + let args = parse_low_raw(["-q"]).unwrap(); + assert_eq!(true, args.quiet); + + // flags like -l and --json cannot override -q, regardless of order + let args = parse_low_raw(["-q", "--json"]).unwrap(); + assert_eq!(true, args.quiet); + + let args = parse_low_raw(["-q", "--files-with-matches"]).unwrap(); + assert_eq!(true, args.quiet); + + let args = parse_low_raw(["-q", "--files-without-match"]).unwrap(); + assert_eq!(true, args.quiet); + + let args = parse_low_raw(["-q", "--count"]).unwrap(); + assert_eq!(true, args.quiet); + + let args = parse_low_raw(["-q", "--count-matches"]).unwrap(); + assert_eq!(true, args.quiet); +} + +/// --regex-size-limit +#[derive(Debug)] +struct RegexSizeLimit; + +impl Flag for RegexSizeLimit { + fn is_switch(&self) -> bool { + false + } + fn name_long(&self) -> &'static str { + "regex-size-limit" + } + fn doc_variable(&self) -> Option<&'static str> { + Some("NUM+SUFFIX?") + } + fn doc_category(&self) -> Category { + Category::Search + } + fn doc_short(&self) -> &'static str { + r"The size limit of the compiled regex." + } + fn doc_long(&self) -> &'static str { + r" +The size limit of the compiled regex, where the compiled regex generally +corresponds to a single object in memory that can match all of the patterns +provided to ripgrep. The default limit is generous enough that most reasonable +patterns (or even a small number of them) should fit. +.sp +This useful to change when you explicitly want to let ripgrep spend potentially +much more time and/or memory building a regex matcher. +.sp +The input format accepts suffixes of \fBK\fP, \fBM\fP or \fBG\fP which +correspond to kilobytes, megabytes and gigabytes, respectively. If no suffix is +provided the input is treated as bytes. +" + } + + fn update(&self, v: FlagValue, args: &mut LowArgs) -> anyhow::Result<()> { + let v = v.unwrap_value(); + args.regex_size_limit = Some(convert::human_readable_usize(&v)?); + Ok(()) + } +} + +#[cfg(test)] +#[test] +fn test_regex_size_limit() { + let args = parse_low_raw(None::<&str>).unwrap(); + assert_eq!(None, args.regex_size_limit); + + #[cfg(target_pointer_width = "64")] + { + let args = parse_low_raw(["--regex-size-limit", "9G"]).unwrap(); + assert_eq!(Some(9 * (1 << 30)), args.regex_size_limit); + + let args = parse_low_raw(["--regex-size-limit=9G"]).unwrap(); + assert_eq!(Some(9 * (1 << 30)), args.regex_size_limit); + + let args = + parse_low_raw(["--regex-size-limit=9G", "--regex-size-limit=0"]) + .unwrap(); + assert_eq!(Some(0), args.regex_size_limit); + } + + let args = parse_low_raw(["--regex-size-limit=0K"]).unwrap(); + assert_eq!(Some(0), args.regex_size_limit); + + let args = parse_low_raw(["--regex-size-limit=0M"]).unwrap(); + assert_eq!(Some(0), args.regex_size_limit); + + let args = parse_low_raw(["--regex-size-limit=0G"]).unwrap(); + assert_eq!(Some(0), args.regex_size_limit); + + let result = + parse_low_raw(["--regex-size-limit", "9999999999999999999999"]); + assert!(result.is_err(), "{result:?}"); + + let result = parse_low_raw(["--regex-size-limit", "9999999999999999G"]); + assert!(result.is_err(), "{result:?}"); +} + +/// -e/--regexp +#[derive(Debug)] +struct Regexp; + +impl Flag for Regexp { + fn is_switch(&self) -> bool { + false + } + fn name_short(&self) -> Option { + Some(b'e') + } + fn name_long(&self) -> &'static str { + "regexp" + } + fn doc_variable(&self) -> Option<&'static str> { + Some("PATTERN") + } + fn doc_category(&self) -> Category { + Category::Input + } + fn doc_short(&self) -> &'static str { + r"A pattern to search for." + } + fn doc_long(&self) -> &'static str { + r" +A pattern to search for. This option can be provided multiple times, where +all patterns given are searched, in addition to any patterns provided by +\flag{file}. Lines matching at least one of the provided patterns are printed. +This flag can also be used when searching for patterns that start with a dash. +.sp +For example, to search for the literal \fB\-foo\fP: +.sp +.EX + rg \-e \-foo +.EE +.sp +You can also use the special \fB\-\-\fP delimiter to indicate that no more +flags will be provided. Namely, the following is equivalent to the above: +.sp +.EX + rg \-\- \-foo +.EE +.sp +When \flag{file} or \flag{regexp} is used, then ripgrep treats all positional +arguments as files or directories to search. +" + } + + fn update(&self, v: FlagValue, args: &mut LowArgs) -> anyhow::Result<()> { + let regexp = convert::string(v.unwrap_value())?; + args.patterns.push(PatternSource::Regexp(regexp)); + Ok(()) + } +} + +#[cfg(test)] +#[test] +fn test_regexp() { + let args = parse_low_raw(None::<&str>).unwrap(); + assert_eq!(Vec::::new(), args.patterns); + + let args = parse_low_raw(["--regexp", "foo"]).unwrap(); + assert_eq!(vec![PatternSource::Regexp("foo".to_string())], args.patterns); + + let args = parse_low_raw(["--regexp=foo"]).unwrap(); + assert_eq!(vec![PatternSource::Regexp("foo".to_string())], args.patterns); + + let args = parse_low_raw(["-e", "foo"]).unwrap(); + assert_eq!(vec![PatternSource::Regexp("foo".to_string())], args.patterns); + + let args = parse_low_raw(["-efoo"]).unwrap(); + assert_eq!(vec![PatternSource::Regexp("foo".to_string())], args.patterns); + + let args = parse_low_raw(["--regexp", "-foo"]).unwrap(); + assert_eq!(vec![PatternSource::Regexp("-foo".to_string())], args.patterns); + + let args = parse_low_raw(["--regexp=-foo"]).unwrap(); + assert_eq!(vec![PatternSource::Regexp("-foo".to_string())], args.patterns); + + let args = parse_low_raw(["-e", "-foo"]).unwrap(); + assert_eq!(vec![PatternSource::Regexp("-foo".to_string())], args.patterns); + + let args = parse_low_raw(["-e-foo"]).unwrap(); + assert_eq!(vec![PatternSource::Regexp("-foo".to_string())], args.patterns); + + let args = parse_low_raw(["--regexp=foo", "--regexp", "bar"]).unwrap(); + assert_eq!( + vec![ + PatternSource::Regexp("foo".to_string()), + PatternSource::Regexp("bar".to_string()) + ], + args.patterns + ); + + // While we support invalid UTF-8 arguments in general, patterns must be + // valid UTF-8. + #[cfg(unix)] + { + use std::{ffi::OsStr, os::unix::ffi::OsStrExt}; + + let bytes = &[b'A', 0xFF, b'Z'][..]; + let result = parse_low_raw([ + OsStr::from_bytes(b"-e"), + OsStr::from_bytes(bytes), + ]); + assert!(result.is_err(), "{result:?}"); + } + + // Check that combining -e/--regexp and -f/--file works as expected. + let args = parse_low_raw(["-efoo", "-fbar"]).unwrap(); + assert_eq!( + vec![ + PatternSource::Regexp("foo".to_string()), + PatternSource::File(PathBuf::from("bar")) + ], + args.patterns + ); + + let args = parse_low_raw(["-efoo", "-fbar", "-equux"]).unwrap(); + assert_eq!( + vec![ + PatternSource::Regexp("foo".to_string()), + PatternSource::File(PathBuf::from("bar")), + PatternSource::Regexp("quux".to_string()), + ], + args.patterns + ); +} + +/// -r/--replace +#[derive(Debug)] +struct Replace; + +impl Flag for Replace { + fn is_switch(&self) -> bool { + false + } + fn name_short(&self) -> Option { + Some(b'r') + } + fn name_long(&self) -> &'static str { + "replace" + } + fn doc_variable(&self) -> Option<&'static str> { + Some("REPLACEMENT") + } + fn doc_category(&self) -> Category { + Category::Output + } + fn doc_short(&self) -> &'static str { + r"Replace matches with the given text." + } + fn doc_long(&self) -> &'static str { + r#" +Replaces every match with the text given when printing results. Neither this +flag nor any other ripgrep flag will modify your files. +.sp +Capture group indices (e.g., \fB$\fP\fI5\fP) and names (e.g., \fB$\fP\fIfoo\fP) +are supported in the replacement string. Capture group indices are numbered +based on the position of the opening parenthesis of the group, where the +leftmost such group is \fB$\fP\fI1\fP. The special \fB$\fP\fI0\fP group +corresponds to the entire match. +.sp +The name of a group is formed by taking the longest string of letters, numbers +and underscores (i.e. \fB[_0-9A-Za-z]\fP) after the \fB$\fP. For example, +\fB$\fP\fI1a\fP will be replaced with the group named \fI1a\fP, not the +group at index \fI1\fP. If the group's name contains characters that aren't +letters, numbers or underscores, or you want to immediately follow the group +with another string, the name should be put inside braces. For example, +\fB${\fP\fI1\fP\fB}\fP\fIa\fP will take the content of the group at index +\fI1\fP and append \fIa\fP to the end of it. +.sp +If an index or name does not refer to a valid capture group, it will be +replaced with an empty string. +.sp +In shells such as Bash and zsh, you should wrap the pattern in single quotes +instead of double quotes. Otherwise, capture group indices will be replaced by +expanded shell variables which will most likely be empty. +.sp +To write a literal \fB$\fP, use \fB$$\fP. +.sp +Note that the replacement by default replaces each match, and not the entire +line. To replace the entire line, you should match the entire line. +.sp +This flag can be used with the \flag{only-matching} flag. +"# + } + + fn update(&self, v: FlagValue, args: &mut LowArgs) -> anyhow::Result<()> { + args.replace = Some(convert::string(v.unwrap_value())?.into()); + Ok(()) + } +} + +#[cfg(test)] +#[test] +fn test_replace() { + use bstr::BString; + + let args = parse_low_raw(None::<&str>).unwrap(); + assert_eq!(None, args.replace); + + let args = parse_low_raw(["--replace", "foo"]).unwrap(); + assert_eq!(Some(BString::from("foo")), args.replace); + + let args = parse_low_raw(["--replace", "-foo"]).unwrap(); + assert_eq!(Some(BString::from("-foo")), args.replace); + + let args = parse_low_raw(["-r", "foo"]).unwrap(); + assert_eq!(Some(BString::from("foo")), args.replace); + + let args = parse_low_raw(["-r", "foo", "-rbar"]).unwrap(); + assert_eq!(Some(BString::from("bar")), args.replace); + + let args = parse_low_raw(["-r", "foo", "-r", ""]).unwrap(); + assert_eq!(Some(BString::from("")), args.replace); +} + +/// -z/--search-zip +#[derive(Debug)] +struct SearchZip; + +impl Flag for SearchZip { + fn is_switch(&self) -> bool { + true + } + fn name_short(&self) -> Option { + Some(b'z') + } + fn name_long(&self) -> &'static str { + "search-zip" + } + fn name_negated(&self) -> Option<&'static str> { + Some("no-search-zip") + } + fn doc_category(&self) -> Category { + Category::Input + } + fn doc_short(&self) -> &'static str { + r"Search in compressed files." + } + fn doc_long(&self) -> &'static str { + r" +This flag instructs ripgrep to search in compressed files. Currently gzip, +bzip2, xz, LZ4, LZMA, Brotli and Zstd files are supported. This option expects +the decompression binaries (such as \fBgzip\fP) to be available in your +\fBPATH\fP. If the required binaries are not found, then ripgrep will not +emit an error messages by default. Use the \flag{debug} flag to see more +information. +.sp +Note that this flag does not make ripgrep search archive formats as directory +trees. It only makes ripgrep detect compressed files and then decompress them +before searching their contents as it would any other file. +.sp +This overrides the \flag{pre} flag. +" + } + + fn update(&self, v: FlagValue, args: &mut LowArgs) -> anyhow::Result<()> { + args.search_zip = if v.unwrap_switch() { + args.pre = None; + true + } else { + false + }; + Ok(()) + } +} + +#[cfg(test)] +#[test] +fn test_search_zip() { + let args = parse_low_raw(None::<&str>).unwrap(); + assert_eq!(false, args.search_zip); + + let args = parse_low_raw(["--search-zip"]).unwrap(); + assert_eq!(true, args.search_zip); + + let args = parse_low_raw(["-z"]).unwrap(); + assert_eq!(true, args.search_zip); + + let args = parse_low_raw(["-z", "--no-search-zip"]).unwrap(); + assert_eq!(false, args.search_zip); + + let args = parse_low_raw(["--pre=foo", "--no-search-zip"]).unwrap(); + assert_eq!(Some(PathBuf::from("foo")), args.pre); + assert_eq!(false, args.search_zip); + + let args = parse_low_raw(["--pre=foo", "--search-zip"]).unwrap(); + assert_eq!(None, args.pre); + assert_eq!(true, args.search_zip); + + let args = parse_low_raw(["--pre=foo", "-z", "--no-search-zip"]).unwrap(); + assert_eq!(None, args.pre); + assert_eq!(false, args.search_zip); +} + +/// -S/--smart-case +#[derive(Debug)] +struct SmartCase; + +impl Flag for SmartCase { + fn is_switch(&self) -> bool { + true + } + fn name_short(&self) -> Option { + Some(b'S') + } + fn name_long(&self) -> &'static str { + "smart-case" + } + fn doc_category(&self) -> Category { + Category::Search + } + fn doc_short(&self) -> &'static str { + r"Smart case search." + } + fn doc_long(&self) -> &'static str { + r" +This flag instructs ripgrep to searches case insensitively if the pattern is +all lowercase. Otherwise, ripgrep will search case sensitively. +.sp +A pattern is considered all lowercase if both of the following rules hold: +.sp +.IP \(bu 3n +First, the pattern contains at least one literal character. For example, +\fBa\\w\fP contains a literal (\fBa\fP) but just \fB\\w\fP does not. +.sp +.IP \(bu 3n +Second, of the literals in the pattern, none of them are considered to be +uppercase according to Unicode. For example, \fBfoo\\pL\fP has no uppercase +literals but \fBFoo\\pL\fP does. +.PP +This overrides the \flag{case-sensitive} and \flag{ignore-case} flags. +" + } + + fn update(&self, v: FlagValue, args: &mut LowArgs) -> anyhow::Result<()> { + assert!(v.unwrap_switch(), "--smart-case flag has no negation"); + args.case = CaseMode::Smart; + Ok(()) + } +} + +#[cfg(test)] +#[test] +fn test_smart_case() { + let args = parse_low_raw(None::<&str>).unwrap(); + assert_eq!(CaseMode::Sensitive, args.case); + + let args = parse_low_raw(["--smart-case"]).unwrap(); + assert_eq!(CaseMode::Smart, args.case); + + let args = parse_low_raw(["-S"]).unwrap(); + assert_eq!(CaseMode::Smart, args.case); + + let args = parse_low_raw(["-S", "-s"]).unwrap(); + assert_eq!(CaseMode::Sensitive, args.case); + + let args = parse_low_raw(["-S", "-i"]).unwrap(); + assert_eq!(CaseMode::Insensitive, args.case); + + let args = parse_low_raw(["-s", "-S"]).unwrap(); + assert_eq!(CaseMode::Smart, args.case); + + let args = parse_low_raw(["-i", "-S"]).unwrap(); + assert_eq!(CaseMode::Smart, args.case); +} + +/// --sort-files +#[derive(Debug)] +struct SortFiles; + +impl Flag for SortFiles { + fn is_switch(&self) -> bool { + true + } + fn name_long(&self) -> &'static str { + "sort-files" + } + fn name_negated(&self) -> Option<&'static str> { + Some("no-sort-files") + } + fn doc_category(&self) -> Category { + Category::Output + } + fn doc_short(&self) -> &'static str { + r"(DEPRECATED) Sort results by file path." + } + fn doc_long(&self) -> &'static str { + r" +DEPRECATED. Use \fB\-\-sort=path\fP instead. +.sp +This flag instructs ripgrep to sort search results by file path +lexicographically in ascending order. Note that this currently disables all +parallelism and runs search in a single thread. +.sp +This flag overrides \flag{sort} and \flag{sortr}. +" + } + + fn update(&self, v: FlagValue, args: &mut LowArgs) -> anyhow::Result<()> { + args.sort = if v.unwrap_switch() { + Some(SortMode { reverse: false, kind: SortModeKind::Path }) + } else { + None + }; + Ok(()) + } +} + +#[cfg(test)] +#[test] +fn test_sort_files() { + let args = parse_low_raw(None::<&str>).unwrap(); + assert_eq!(None, args.sort); + + let args = parse_low_raw(["--sort-files"]).unwrap(); + assert_eq!( + Some(SortMode { reverse: false, kind: SortModeKind::Path }), + args.sort + ); + + let args = parse_low_raw(["--sort-files", "--no-sort-files"]).unwrap(); + assert_eq!(None, args.sort); + + let args = parse_low_raw(["--sort", "created", "--sort-files"]).unwrap(); + assert_eq!( + Some(SortMode { reverse: false, kind: SortModeKind::Path }), + args.sort + ); + + let args = parse_low_raw(["--sort-files", "--sort", "created"]).unwrap(); + assert_eq!( + Some(SortMode { reverse: false, kind: SortModeKind::Created }), + args.sort + ); + + let args = parse_low_raw(["--sortr", "created", "--sort-files"]).unwrap(); + assert_eq!( + Some(SortMode { reverse: false, kind: SortModeKind::Path }), + args.sort + ); + + let args = parse_low_raw(["--sort-files", "--sortr", "created"]).unwrap(); + assert_eq!( + Some(SortMode { reverse: true, kind: SortModeKind::Created }), + args.sort + ); + + let args = parse_low_raw(["--sort=path", "--no-sort-files"]).unwrap(); + assert_eq!(None, args.sort); + + let args = parse_low_raw(["--sortr=path", "--no-sort-files"]).unwrap(); + assert_eq!(None, args.sort); +} + +/// --sort +#[derive(Debug)] +struct Sort; + +impl Flag for Sort { + fn is_switch(&self) -> bool { + false + } + fn name_long(&self) -> &'static str { + "sort" + } + fn doc_variable(&self) -> Option<&'static str> { + Some("SORTBY") + } + fn doc_category(&self) -> Category { + Category::Output + } + fn doc_short(&self) -> &'static str { + r"Sort results in ascending order." + } + fn doc_long(&self) -> &'static str { + r" +This flag enables sorting of results in ascending order. The possible values +for this flag are: +.sp +.TP 12 +\fBnone\fP +(Default) Do not sort results. Fastest. Can be multi-threaded. +.TP 12 +\fBpath\fP +Sort by file path. Always single-threaded. The order is determined by sorting +files in each directory entry during traversal. This means that given the files +\fBa/b\fP and \fBa+\fP, the latter will sort after the former even though +\fB+\fP would normally sort before \fB/\fP. +.TP 12 +\fBmodified\fP +Sort by the last modified time on a file. Always single-threaded. +.TP 12 +\fBaccessed\fP +Sort by the last accessed time on a file. Always single-threaded. +.TP 12 +\fBcreated\fP +Sort by the creation time on a file. Always single-threaded. +.PP +If the chosen (manually or by-default) sorting criteria isn't available on your +system (for example, creation time is not available on ext4 file systems), then +ripgrep will attempt to detect this, print an error and exit without searching. +.sp +To sort results in reverse or descending order, use the \flag{sortr} flag. +Also, this flag overrides \flag{sortr}. +.sp +Note that sorting results currently always forces ripgrep to abandon +parallelism and run in a single thread. +" + } + fn doc_choices(&self) -> &'static [&'static str] { + &["none", "path", "modified", "accessed", "created"] + } + + fn update(&self, v: FlagValue, args: &mut LowArgs) -> anyhow::Result<()> { + let kind = match convert::str(&v.unwrap_value())? { + "none" => { + args.sort = None; + return Ok(()); + } + "path" => SortModeKind::Path, + "modified" => SortModeKind::LastModified, + "accessed" => SortModeKind::LastAccessed, + "created" => SortModeKind::Created, + unk => anyhow::bail!("choice '{unk}' is unrecognized"), + }; + args.sort = Some(SortMode { reverse: false, kind }); + Ok(()) + } +} + +#[cfg(test)] +#[test] +fn test_sort() { + let args = parse_low_raw(None::<&str>).unwrap(); + assert_eq!(None, args.sort); + + let args = parse_low_raw(["--sort", "path"]).unwrap(); + assert_eq!( + Some(SortMode { reverse: false, kind: SortModeKind::Path }), + args.sort + ); + + let args = parse_low_raw(["--sort", "path", "--sort=created"]).unwrap(); + assert_eq!( + Some(SortMode { reverse: false, kind: SortModeKind::Created }), + args.sort + ); + + let args = parse_low_raw(["--sort=none"]).unwrap(); + assert_eq!(None, args.sort); + + let args = parse_low_raw(["--sort", "path", "--sort=none"]).unwrap(); + assert_eq!(None, args.sort); +} + +/// --sortr +#[derive(Debug)] +struct Sortr; + +impl Flag for Sortr { + fn is_switch(&self) -> bool { + false + } + fn name_long(&self) -> &'static str { + "sortr" + } + fn doc_variable(&self) -> Option<&'static str> { + Some("SORTBY") + } + fn doc_category(&self) -> Category { + Category::Output + } + fn doc_short(&self) -> &'static str { + r"Sort results in descending order." + } + fn doc_long(&self) -> &'static str { + r" +This flag enables sorting of results in descending order. The possible values +for this flag are: +.sp +.TP 12 +\fBnone\fP +(Default) Do not sort results. Fastest. Can be multi-threaded. +.TP 12 +\fBpath\fP +Sort by file path. Always single-threaded. The order is determined by sorting +files in each directory entry during traversal. This means that given the files +\fBa/b\fP and \fBa+\fP, the latter will sort before the former even though +\fB+\fP would normally sort after \fB/\fP when doing a reverse lexicographic +sort. +.TP 12 +\fBmodified\fP +Sort by the last modified time on a file. Always single-threaded. +.TP 12 +\fBaccessed\fP +Sort by the last accessed time on a file. Always single-threaded. +.TP 12 +\fBcreated\fP +Sort by the creation time on a file. Always single-threaded. +.PP +If the chosen (manually or by-default) sorting criteria isn't available on your +system (for example, creation time is not available on ext4 file systems), then +ripgrep will attempt to detect this, print an error and exit without searching. +.sp +To sort results in ascending order, use the \flag{sort} flag. Also, this flag +overrides \flag{sort}. +.sp +Note that sorting results currently always forces ripgrep to abandon +parallelism and run in a single thread. +" + } + fn doc_choices(&self) -> &'static [&'static str] { + &["none", "path", "modified", "accessed", "created"] + } + + fn update(&self, v: FlagValue, args: &mut LowArgs) -> anyhow::Result<()> { + let kind = match convert::str(&v.unwrap_value())? { + "none" => { + args.sort = None; + return Ok(()); + } + "path" => SortModeKind::Path, + "modified" => SortModeKind::LastModified, + "accessed" => SortModeKind::LastAccessed, + "created" => SortModeKind::Created, + unk => anyhow::bail!("choice '{unk}' is unrecognized"), + }; + args.sort = Some(SortMode { reverse: true, kind }); + Ok(()) + } +} + +#[cfg(test)] +#[test] +fn test_sortr() { + let args = parse_low_raw(None::<&str>).unwrap(); + assert_eq!(None, args.sort); + + let args = parse_low_raw(["--sortr", "path"]).unwrap(); + assert_eq!( + Some(SortMode { reverse: true, kind: SortModeKind::Path }), + args.sort + ); + + let args = parse_low_raw(["--sortr", "path", "--sortr=created"]).unwrap(); + assert_eq!( + Some(SortMode { reverse: true, kind: SortModeKind::Created }), + args.sort + ); + + let args = parse_low_raw(["--sortr=none"]).unwrap(); + assert_eq!(None, args.sort); + + let args = parse_low_raw(["--sortr", "path", "--sortr=none"]).unwrap(); + assert_eq!(None, args.sort); + + let args = parse_low_raw(["--sort=path", "--sortr=path"]).unwrap(); + assert_eq!( + Some(SortMode { reverse: true, kind: SortModeKind::Path }), + args.sort + ); + + let args = parse_low_raw(["--sortr=path", "--sort=path"]).unwrap(); + assert_eq!( + Some(SortMode { reverse: false, kind: SortModeKind::Path }), + args.sort + ); +} + +/// --stats +#[derive(Debug)] +struct Stats; + +impl Flag for Stats { + fn is_switch(&self) -> bool { + true + } + fn name_long(&self) -> &'static str { + "stats" + } + fn name_negated(&self) -> Option<&'static str> { + Some("no-stats") + } + fn doc_category(&self) -> Category { + Category::Logging + } + fn doc_short(&self) -> &'static str { + r"Print statistics about the search." + } + fn doc_long(&self) -> &'static str { + r" +When enabled, ripgrep will print aggregate statistics about the search. When +this flag is present, ripgrep will print at least the following stats to +stdout at the end of the search: number of matched lines, number of files with +matches, number of files searched, and the time taken for the entire search to +complete. +.sp +This set of aggregate statistics may expand over time. +.sp +This flag is always and implicitly enabled when \flag{json} is used. +.sp +Note that this flag has no effect if \flag{files}, \flag{files-with-matches} or +\flag{files-without-match} is passed. +" + } + + fn update(&self, v: FlagValue, args: &mut LowArgs) -> anyhow::Result<()> { + args.stats = v.unwrap_switch(); + Ok(()) + } +} + +#[cfg(test)] +#[test] +fn test_stats() { + let args = parse_low_raw(None::<&str>).unwrap(); + assert_eq!(false, args.stats); + + let args = parse_low_raw(["--stats"]).unwrap(); + assert_eq!(true, args.stats); + + let args = parse_low_raw(["--stats", "--no-stats"]).unwrap(); + assert_eq!(false, args.stats); +} + +/// --stop-on-nonmatch +#[derive(Debug)] +struct StopOnNonmatch; + +impl Flag for StopOnNonmatch { + fn is_switch(&self) -> bool { + true + } + fn name_long(&self) -> &'static str { + "stop-on-nonmatch" + } + fn doc_category(&self) -> Category { + Category::Search + } + fn doc_short(&self) -> &'static str { + r"Stop searching after a non-match." + } + fn doc_long(&self) -> &'static str { + r" +Enabling this option will cause ripgrep to stop reading a file once it +encounters a non-matching line after it has encountered a matching line. +This is useful if it is expected that all matches in a given file will be on +sequential lines, for example due to the lines being sorted. +.sp +This overrides the \flag{multiline} flag. +" + } + + fn update(&self, v: FlagValue, args: &mut LowArgs) -> anyhow::Result<()> { + assert!(v.unwrap_switch(), "--stop-on-nonmatch has no negation"); + args.stop_on_nonmatch = true; + args.multiline = false; + Ok(()) + } +} + +#[cfg(test)] +#[test] +fn test_stop_on_nonmatch() { + let args = parse_low_raw(None::<&str>).unwrap(); + assert_eq!(false, args.stop_on_nonmatch); + + let args = parse_low_raw(["--stop-on-nonmatch"]).unwrap(); + assert_eq!(true, args.stop_on_nonmatch); + + let args = parse_low_raw(["--stop-on-nonmatch", "-U"]).unwrap(); + assert_eq!(true, args.multiline); + assert_eq!(false, args.stop_on_nonmatch); + + let args = parse_low_raw(["-U", "--stop-on-nonmatch"]).unwrap(); + assert_eq!(false, args.multiline); + assert_eq!(true, args.stop_on_nonmatch); + + let args = + parse_low_raw(["--stop-on-nonmatch", "--no-multiline"]).unwrap(); + assert_eq!(false, args.multiline); + assert_eq!(true, args.stop_on_nonmatch); +} + +/// -a/--text +#[derive(Debug)] +struct Text; + +impl Flag for Text { + fn is_switch(&self) -> bool { + true + } + fn name_short(&self) -> Option { + Some(b'a') + } + fn name_long(&self) -> &'static str { + "text" + } + fn name_negated(&self) -> Option<&'static str> { + Some("no-text") + } + fn doc_category(&self) -> Category { + Category::Search + } + fn doc_short(&self) -> &'static str { + r"Search binary files as if they were text." + } + fn doc_long(&self) -> &'static str { + r" +This flag instructs ripgrep to search binary files as if they were text. When +this flag is present, ripgrep's binary file detection is disabled. This means +that when a binary file is searched, its contents may be printed if there is +a match. This may cause escape codes to be printed that alter the behavior of +your terminal. +.sp +When binary file detection is enabled, it is imperfect. In general, it uses +a simple heuristic. If a \fBNUL\fP byte is seen during search, then the file +is considered binary and searching stops (unless this flag is present). +Alternatively, if the \flag{binary} flag is used, then ripgrep will only quit +when it sees a \fBNUL\fP byte after it sees a match (or searches the entire +file). +.sp +This flag overrides the \flag{binary} flag. +" + } + + fn update(&self, v: FlagValue, args: &mut LowArgs) -> anyhow::Result<()> { + args.binary = if v.unwrap_switch() { + BinaryMode::AsText + } else { + BinaryMode::Auto + }; + Ok(()) + } +} + +#[cfg(test)] +#[test] +fn test_text() { + let args = parse_low_raw(None::<&str>).unwrap(); + assert_eq!(BinaryMode::Auto, args.binary); + + let args = parse_low_raw(["--text"]).unwrap(); + assert_eq!(BinaryMode::AsText, args.binary); + + let args = parse_low_raw(["-a"]).unwrap(); + assert_eq!(BinaryMode::AsText, args.binary); + + let args = parse_low_raw(["-a", "--no-text"]).unwrap(); + assert_eq!(BinaryMode::Auto, args.binary); + + let args = parse_low_raw(["-a", "--binary"]).unwrap(); + assert_eq!(BinaryMode::SearchAndSuppress, args.binary); + + let args = parse_low_raw(["--binary", "-a"]).unwrap(); + assert_eq!(BinaryMode::AsText, args.binary); + + let args = parse_low_raw(["-a", "--no-binary"]).unwrap(); + assert_eq!(BinaryMode::Auto, args.binary); + + let args = parse_low_raw(["--binary", "--no-text"]).unwrap(); + assert_eq!(BinaryMode::Auto, args.binary); +} + +/// -j/--threads +#[derive(Debug)] +struct Threads; + +impl Flag for Threads { + fn is_switch(&self) -> bool { + false + } + fn name_short(&self) -> Option { + Some(b'j') + } + fn name_long(&self) -> &'static str { + "threads" + } + fn doc_variable(&self) -> Option<&'static str> { + Some("NUM") + } + fn doc_category(&self) -> Category { + Category::Search + } + fn doc_short(&self) -> &'static str { + r"Set the approximate number of threads to use." + } + fn doc_long(&self) -> &'static str { + r" +This flag sets the approximate number of threads to use. A value of \fB0\fP +(which is the default) causes ripgrep to choose the thread count using +heuristics. +" + } + + fn update(&self, v: FlagValue, args: &mut LowArgs) -> anyhow::Result<()> { + let threads = convert::usize(&v.unwrap_value())?; + args.threads = if threads == 0 { None } else { Some(threads) }; + Ok(()) + } +} + +#[cfg(test)] +#[test] +fn test_threads() { + let args = parse_low_raw(None::<&str>).unwrap(); + assert_eq!(None, args.threads); + + let args = parse_low_raw(["--threads", "5"]).unwrap(); + assert_eq!(Some(5), args.threads); + + let args = parse_low_raw(["-j", "5"]).unwrap(); + assert_eq!(Some(5), args.threads); + + let args = parse_low_raw(["-j5"]).unwrap(); + assert_eq!(Some(5), args.threads); + + let args = parse_low_raw(["-j5", "-j10"]).unwrap(); + assert_eq!(Some(10), args.threads); + + let args = parse_low_raw(["-j5", "-j0"]).unwrap(); + assert_eq!(None, args.threads); +} + +/// --trace +#[derive(Debug)] +struct Trace; + +impl Flag for Trace { + fn is_switch(&self) -> bool { + true + } + fn name_long(&self) -> &'static str { + "trace" + } + fn doc_category(&self) -> Category { + Category::Logging + } + fn doc_short(&self) -> &'static str { + r"Show trace messages." + } + fn doc_long(&self) -> &'static str { + r" +Show trace messages. This shows even more detail than the \flag{debug} +flag. Generally, one should only use this if \flag{debug} doesn't emit the +information you're looking for. +" + } + + fn update(&self, v: FlagValue, args: &mut LowArgs) -> anyhow::Result<()> { + assert!(v.unwrap_switch(), "--trace can only be enabled"); + args.logging = Some(LoggingMode::Trace); + Ok(()) + } +} + +#[cfg(test)] +#[test] +fn test_trace() { + let args = parse_low_raw(None::<&str>).unwrap(); + assert_eq!(None, args.logging); + + let args = parse_low_raw(["--trace"]).unwrap(); + assert_eq!(Some(LoggingMode::Trace), args.logging); + + let args = parse_low_raw(["--debug", "--trace"]).unwrap(); + assert_eq!(Some(LoggingMode::Trace), args.logging); +} + +/// --trim +#[derive(Debug)] +struct Trim; + +impl Flag for Trim { + fn is_switch(&self) -> bool { + true + } + fn name_long(&self) -> &'static str { + "trim" + } + fn name_negated(&self) -> Option<&'static str> { + Some("no-trim") + } + fn doc_category(&self) -> Category { + Category::Output + } + fn doc_short(&self) -> &'static str { + r"Trim prefix whitespace from matches." + } + fn doc_long(&self) -> &'static str { + r" +When set, all ASCII whitespace at the beginning of each line printed will be +removed. +" + } + + fn update(&self, v: FlagValue, args: &mut LowArgs) -> anyhow::Result<()> { + args.trim = v.unwrap_switch(); + Ok(()) + } +} + +#[cfg(test)] +#[test] +fn test_trim() { + let args = parse_low_raw(None::<&str>).unwrap(); + assert_eq!(false, args.trim); + + let args = parse_low_raw(["--trim"]).unwrap(); + assert_eq!(true, args.trim); + + let args = parse_low_raw(["--trim", "--no-trim"]).unwrap(); + assert_eq!(false, args.trim); +} + +/// -t/--type +#[derive(Debug)] +struct Type; + +impl Flag for Type { + fn is_switch(&self) -> bool { + false + } + fn name_short(&self) -> Option { + Some(b't') + } + fn name_long(&self) -> &'static str { + "type" + } + fn doc_variable(&self) -> Option<&'static str> { + Some("TYPE") + } + fn doc_category(&self) -> Category { + Category::Filter + } + fn doc_short(&self) -> &'static str { + r"Only search files matching TYPE." + } + fn doc_long(&self) -> &'static str { + r#" +This flag limits ripgrep to searching files matching \fITYPE\fP. Multiple +\flag{type} flags may be provided. +.sp +This flag supports the special value \fBall\fP, which will behave as if +\flag{type} was provided for every file type supported by ripgrep (including +any custom file types). The end result is that \fB\-\-type=all\fP causes +ripgrep to search in "whitelist" mode, where it will only search files it +recognizes via its type definitions. +.sp +Note that this flag has lower precedence than both the \flag{glob} flag and +any rules found in ignore files. +.sp +To see the list of available file types, use the \flag{type-list} flag. +"# + } + fn completion_type(&self) -> CompletionType { + CompletionType::Filetype + } + + fn update(&self, v: FlagValue, args: &mut LowArgs) -> anyhow::Result<()> { + args.type_changes.push(TypeChange::Select { + name: convert::string(v.unwrap_value())?, + }); + Ok(()) + } +} + +#[cfg(test)] +#[test] +fn test_type() { + let select = |name: &str| TypeChange::Select { name: name.to_string() }; + + let args = parse_low_raw(None::<&str>).unwrap(); + assert_eq!(Vec::::new(), args.type_changes); + + let args = parse_low_raw(["--type", "rust"]).unwrap(); + assert_eq!(vec![select("rust")], args.type_changes); + + let args = parse_low_raw(["-t", "rust"]).unwrap(); + assert_eq!(vec![select("rust")], args.type_changes); + + let args = parse_low_raw(["-trust"]).unwrap(); + assert_eq!(vec![select("rust")], args.type_changes); + + let args = parse_low_raw(["-trust", "-tpython"]).unwrap(); + assert_eq!(vec![select("rust"), select("python")], args.type_changes); + + let args = parse_low_raw(["-tabcdefxyz"]).unwrap(); + assert_eq!(vec![select("abcdefxyz")], args.type_changes); +} + +/// --type-add +#[derive(Debug)] +struct TypeAdd; + +impl Flag for TypeAdd { + fn is_switch(&self) -> bool { + false + } + fn name_long(&self) -> &'static str { + "type-add" + } + fn doc_variable(&self) -> Option<&'static str> { + Some("TYPESPEC") + } + fn doc_category(&self) -> Category { + Category::Filter + } + fn doc_short(&self) -> &'static str { + r"Add a new glob for a file type." + } + fn doc_long(&self) -> &'static str { + r" +This flag adds a new glob for a particular file type. Only one glob can be +added at a time. Multiple \flag{type-add} flags can be provided. Unless +\flag{type-clear} is used, globs are added to any existing globs defined inside +of ripgrep. +.sp +Note that this must be passed to every invocation of ripgrep. Type settings are +not persisted. See \fBCONFIGURATION FILES\fP for a workaround. +.sp +Example: +.sp +.EX + rg \-\-type\-add 'foo:*.foo' -tfoo \fIPATTERN\fP +.EE +.sp +This flag can also be used to include rules from other types with the special +include directive. The include directive permits specifying one or more other +type names (separated by a comma) that have been defined and its rules will +automatically be imported into the type specified. For example, to create a +type called src that matches C++, Python and Markdown files, one can use: +.sp +.EX + \-\-type\-add 'src:include:cpp,py,md' +.EE +.sp +Additional glob rules can still be added to the src type by using this flag +again: +.sp +.EX + \-\-type\-add 'src:include:cpp,py,md' \-\-type\-add 'src:*.foo' +.EE +.sp +Note that type names must consist only of Unicode letters or numbers. +Punctuation characters are not allowed. +" + } + + fn update(&self, v: FlagValue, args: &mut LowArgs) -> anyhow::Result<()> { + args.type_changes + .push(TypeChange::Add { def: convert::string(v.unwrap_value())? }); + Ok(()) + } +} + +#[cfg(test)] +#[test] +fn test_type_add() { + let args = parse_low_raw(None::<&str>).unwrap(); + assert_eq!(Vec::::new(), args.type_changes); + + let args = parse_low_raw(["--type-add", "foo"]).unwrap(); + assert_eq!( + vec![TypeChange::Add { def: "foo".to_string() }], + args.type_changes + ); + + let args = parse_low_raw(["--type-add", "foo", "--type-add=bar"]).unwrap(); + assert_eq!( + vec![ + TypeChange::Add { def: "foo".to_string() }, + TypeChange::Add { def: "bar".to_string() } + ], + args.type_changes + ); +} + +/// --type-clear +#[derive(Debug)] +struct TypeClear; + +impl Flag for TypeClear { + fn is_switch(&self) -> bool { + false + } + fn name_long(&self) -> &'static str { + "type-clear" + } + fn doc_variable(&self) -> Option<&'static str> { + Some("TYPE") + } + fn doc_category(&self) -> Category { + Category::Filter + } + fn doc_short(&self) -> &'static str { + r"Clear globs for a file type." + } + fn doc_long(&self) -> &'static str { + r" +Clear the file type globs previously defined for \fITYPE\fP. This clears any +previously defined globs for the \fITYPE\fP, but globs can be added after this +flag. +.sp +Note that this must be passed to every invocation of ripgrep. Type settings are +not persisted. See \fBCONFIGURATION FILES\fP for a workaround. +" + } + + fn update(&self, v: FlagValue, args: &mut LowArgs) -> anyhow::Result<()> { + args.type_changes.push(TypeChange::Clear { + name: convert::string(v.unwrap_value())?, + }); + Ok(()) + } +} + +#[cfg(test)] +#[test] +fn test_type_clear() { + let args = parse_low_raw(None::<&str>).unwrap(); + assert_eq!(Vec::::new(), args.type_changes); + + let args = parse_low_raw(["--type-clear", "foo"]).unwrap(); + assert_eq!( + vec![TypeChange::Clear { name: "foo".to_string() }], + args.type_changes + ); + + let args = + parse_low_raw(["--type-clear", "foo", "--type-clear=bar"]).unwrap(); + assert_eq!( + vec![ + TypeChange::Clear { name: "foo".to_string() }, + TypeChange::Clear { name: "bar".to_string() } + ], + args.type_changes + ); +} + +/// --type-not +#[derive(Debug)] +struct TypeNot; + +impl Flag for TypeNot { + fn is_switch(&self) -> bool { + false + } + fn name_short(&self) -> Option { + Some(b'T') + } + fn name_long(&self) -> &'static str { + "type-not" + } + fn doc_variable(&self) -> Option<&'static str> { + Some("TYPE") + } + fn doc_category(&self) -> Category { + Category::Filter + } + fn doc_short(&self) -> &'static str { + r"Do not search files matching TYPE." + } + fn doc_long(&self) -> &'static str { + r#" +Do not search files matching \fITYPE\fP. Multiple \flag{type-not} flags may be +provided. Use the \flag{type-list} flag to list all available types. +.sp +This flag supports the special value \fBall\fP, which will behave +as if \flag{type-not} was provided for every file type supported by +ripgrep (including any custom file types). The end result is that +\fB\-\-type\-not=all\fP causes ripgrep to search in "blacklist" mode, where it +will only search files that are unrecognized by its type definitions. +.sp +To see the list of available file types, use the \flag{type-list} flag. +"# + } + fn completion_type(&self) -> CompletionType { + CompletionType::Filetype + } + + fn update(&self, v: FlagValue, args: &mut LowArgs) -> anyhow::Result<()> { + args.type_changes.push(TypeChange::Negate { + name: convert::string(v.unwrap_value())?, + }); + Ok(()) + } +} + +#[cfg(test)] +#[test] +fn test_type_not() { + let select = |name: &str| TypeChange::Select { name: name.to_string() }; + let negate = |name: &str| TypeChange::Negate { name: name.to_string() }; + + let args = parse_low_raw(None::<&str>).unwrap(); + assert_eq!(Vec::::new(), args.type_changes); + + let args = parse_low_raw(["--type-not", "rust"]).unwrap(); + assert_eq!(vec![negate("rust")], args.type_changes); + + let args = parse_low_raw(["-T", "rust"]).unwrap(); + assert_eq!(vec![negate("rust")], args.type_changes); + + let args = parse_low_raw(["-Trust"]).unwrap(); + assert_eq!(vec![negate("rust")], args.type_changes); + + let args = parse_low_raw(["-Trust", "-Tpython"]).unwrap(); + assert_eq!(vec![negate("rust"), negate("python")], args.type_changes); + + let args = parse_low_raw(["-Tabcdefxyz"]).unwrap(); + assert_eq!(vec![negate("abcdefxyz")], args.type_changes); + + let args = parse_low_raw(["-Trust", "-ttoml", "-Tjson"]).unwrap(); + assert_eq!( + vec![negate("rust"), select("toml"), negate("json")], + args.type_changes + ); +} + +/// --type-list +#[derive(Debug)] +struct TypeList; + +impl Flag for TypeList { + fn is_switch(&self) -> bool { + true + } + fn name_long(&self) -> &'static str { + "type-list" + } + fn doc_category(&self) -> Category { + Category::OtherBehaviors + } + fn doc_short(&self) -> &'static str { + r"Show all supported file types." + } + fn doc_long(&self) -> &'static str { + r" +Show all supported file types and their corresponding globs. This takes any +\flag{type-add} and \flag{type-clear} flags given into account. Each type is +printed on its own line, followed by a \fB:\fP and then a comma-delimited list +of globs for that type on the same line. +" + } + + fn update(&self, v: FlagValue, args: &mut LowArgs) -> anyhow::Result<()> { + assert!(v.unwrap_switch(), "--type-list has no negation"); + args.mode.update(Mode::Types); + Ok(()) + } +} + +#[cfg(test)] +#[test] +fn test_type_list() { + let args = parse_low_raw(None::<&str>).unwrap(); + assert_eq!(Mode::Search(SearchMode::Standard), args.mode); + + let args = parse_low_raw(["--type-list"]).unwrap(); + assert_eq!(Mode::Types, args.mode); +} + +/// -u/--unrestricted +#[derive(Debug)] +struct Unrestricted; + +impl Flag for Unrestricted { + fn is_switch(&self) -> bool { + true + } + fn name_short(&self) -> Option { + Some(b'u') + } + fn name_long(&self) -> &'static str { + "unrestricted" + } + fn doc_category(&self) -> Category { + Category::Filter + } + fn doc_short(&self) -> &'static str { + r#"Reduce the level of "smart" filtering."# + } + fn doc_long(&self) -> &'static str { + r#" +This flag reduces the level of "smart" filtering. Repeated uses (up to 3) reduces +the filtering even more. When repeated three times, ripgrep will search every +file in a directory tree. +.sp +A single \flag{unrestricted} flag is equivalent to \flag{no-ignore}. Two +\flag{unrestricted} flags is equivalent to \flag{no-ignore} \flag{hidden}. +Three \flag{unrestricted} flags is equivalent to \flag{no-ignore} \flag{hidden} +\flag{binary}. +.sp +The only filtering ripgrep still does when \fB-uuu\fP is given is to skip +symbolic links and to avoid printing matches from binary files. Symbolic links +can be followed via the \flag{follow} flag, and binary files can be treated as +text files via the \flag{text} flag. +"# + } + + fn update(&self, v: FlagValue, args: &mut LowArgs) -> anyhow::Result<()> { + assert!(v.unwrap_switch(), "--unrestricted has no negation"); + args.unrestricted = args.unrestricted.saturating_add(1); + anyhow::ensure!( + args.unrestricted <= 3, + "flag can only be repeated up to 3 times" + ); + if args.unrestricted == 1 { + NoIgnore.update(FlagValue::Switch(true), args)?; + } else if args.unrestricted == 2 { + Hidden.update(FlagValue::Switch(true), args)?; + } else { + assert_eq!(args.unrestricted, 3); + Binary.update(FlagValue::Switch(true), args)?; + } + Ok(()) + } +} + +#[cfg(test)] +#[test] +fn test_unrestricted() { + let args = parse_low_raw(None::<&str>).unwrap(); + assert_eq!(false, args.no_ignore_vcs); + assert_eq!(false, args.hidden); + assert_eq!(BinaryMode::Auto, args.binary); + + let args = parse_low_raw(["--unrestricted"]).unwrap(); + assert_eq!(true, args.no_ignore_vcs); + assert_eq!(false, args.hidden); + assert_eq!(BinaryMode::Auto, args.binary); + + let args = parse_low_raw(["--unrestricted", "-u"]).unwrap(); + assert_eq!(true, args.no_ignore_vcs); + assert_eq!(true, args.hidden); + assert_eq!(BinaryMode::Auto, args.binary); + + let args = parse_low_raw(["-uuu"]).unwrap(); + assert_eq!(true, args.no_ignore_vcs); + assert_eq!(true, args.hidden); + assert_eq!(BinaryMode::SearchAndSuppress, args.binary); + + let result = parse_low_raw(["-uuuu"]); + assert!(result.is_err(), "{result:?}"); +} + +/// --version +#[derive(Debug)] +struct Version; + +impl Flag for Version { + fn is_switch(&self) -> bool { + true + } + fn name_short(&self) -> Option { + Some(b'V') + } + fn name_long(&self) -> &'static str { + "version" + } + fn doc_category(&self) -> Category { + Category::OtherBehaviors + } + fn doc_short(&self) -> &'static str { + r"Print ripgrep's version." + } + fn doc_long(&self) -> &'static str { + r" +This flag prints ripgrep's version. This also may print other relevant +information, such as the presence of target specific optimizations and the +\fBgit\fP revision that this build of ripgrep was compiled from. +" + } + + fn update(&self, v: FlagValue, _: &mut LowArgs) -> anyhow::Result<()> { + assert!(v.unwrap_switch(), "--version has no negation"); + // Since this flag has different semantics for -V and --version and the + // Flag trait doesn't support encoding this sort of thing, we handle it + // as a special case in the parser. + Ok(()) + } +} + +#[cfg(test)] +#[test] +fn test_version() { + let args = parse_low_raw(None::<&str>).unwrap(); + assert_eq!(None, args.special); + + let args = parse_low_raw(["-V"]).unwrap(); + assert_eq!(Some(SpecialMode::VersionShort), args.special); + + let args = parse_low_raw(["--version"]).unwrap(); + assert_eq!(Some(SpecialMode::VersionLong), args.special); + + let args = parse_low_raw(["-V", "--version"]).unwrap(); + assert_eq!(Some(SpecialMode::VersionLong), args.special); + + let args = parse_low_raw(["--version", "-V"]).unwrap(); + assert_eq!(Some(SpecialMode::VersionShort), args.special); +} + +/// --vimgrep +#[derive(Debug)] +struct Vimgrep; + +impl Flag for Vimgrep { + fn is_switch(&self) -> bool { + true + } + fn name_long(&self) -> &'static str { + "vimgrep" + } + fn doc_category(&self) -> Category { + Category::Output + } + fn doc_short(&self) -> &'static str { + r"Print results in a vim compatible format." + } + fn doc_long(&self) -> &'static str { + r" +This flag instructs ripgrep to print results with every match on its own line, +including line numbers and column numbers. +.sp +With this option, a line with more than one match will be printed in its +entirety more than once. For that reason, the total amount of output as a +result of this flag can be quadratic in the size of the input. For example, +if the pattern matches every byte in an input file, then each line will be +repeated for every byte matched. For this reason, users should only use this +flag when there is no other choice. Editor integrations should prefer some +other way of reading results from ripgrep, such as via the \flag{json} flag. +One alternative to avoiding exorbitant memory usage is to force ripgrep into +single threaded mode with the \flag{threads} flag. Note though that this will +not impact the total size of the output, just the heap memory that ripgrep will +use. +" + } + fn doc_choices(&self) -> &'static [&'static str] { + &[] + } + + fn update(&self, v: FlagValue, args: &mut LowArgs) -> anyhow::Result<()> { + assert!(v.unwrap_switch(), "--vimgrep has no negation"); + args.vimgrep = true; + Ok(()) + } +} + +#[cfg(test)] +#[test] +fn test_vimgrep() { + let args = parse_low_raw(None::<&str>).unwrap(); + assert_eq!(false, args.vimgrep); + + let args = parse_low_raw(["--vimgrep"]).unwrap(); + assert_eq!(true, args.vimgrep); +} + +/// --with-filename +#[derive(Debug)] +struct WithFilename; + +impl Flag for WithFilename { + fn is_switch(&self) -> bool { + true + } + fn name_short(&self) -> Option { + Some(b'H') + } + fn name_long(&self) -> &'static str { + "with-filename" + } + fn doc_category(&self) -> Category { + Category::Output + } + fn doc_short(&self) -> &'static str { + r"Print the file path with each matching line." + } + fn doc_long(&self) -> &'static str { + r" +This flag instructs ripgrep to print the file path for each matching line. +This is the default when more than one file is searched. If \flag{heading} is +enabled (the default when printing to a tty), the file path will be shown above +clusters of matches from each file; otherwise, the file name will be shown as a +prefix for each matched line. +.sp +This flag overrides \flag{no-filename}. +" + } + + fn update(&self, v: FlagValue, args: &mut LowArgs) -> anyhow::Result<()> { + assert!(v.unwrap_switch(), "--with-filename has no defined negation"); + args.with_filename = Some(true); + Ok(()) + } +} + +#[cfg(test)] +#[test] +fn test_with_filename() { + let args = parse_low_raw(None::<&str>).unwrap(); + assert_eq!(None, args.with_filename); + + let args = parse_low_raw(["--with-filename"]).unwrap(); + assert_eq!(Some(true), args.with_filename); + + let args = parse_low_raw(["-H"]).unwrap(); + assert_eq!(Some(true), args.with_filename); +} + +/// --no-filename +#[derive(Debug)] +struct WithFilenameNo; + +impl Flag for WithFilenameNo { + fn is_switch(&self) -> bool { + true + } + fn name_short(&self) -> Option { + Some(b'I') + } + fn name_long(&self) -> &'static str { + "no-filename" + } + fn doc_category(&self) -> Category { + Category::Output + } + fn doc_short(&self) -> &'static str { + r"Never print the path with each matching line." + } + fn doc_long(&self) -> &'static str { + r" +This flag instructs ripgrep to never print the file path with each matching +line. This is the default when ripgrep is explicitly instructed to search one +file or stdin. +.sp +This flag overrides \flag{with-filename}. +" + } + fn doc_choices(&self) -> &'static [&'static str] { + &[] + } + + fn update(&self, v: FlagValue, args: &mut LowArgs) -> anyhow::Result<()> { + assert!(v.unwrap_switch(), "--no-filename has no defined negation"); + args.with_filename = Some(false); + Ok(()) + } +} + +#[cfg(test)] +#[test] +fn test_with_filename_no() { + let args = parse_low_raw(None::<&str>).unwrap(); + assert_eq!(None, args.with_filename); + + let args = parse_low_raw(["--no-filename"]).unwrap(); + assert_eq!(Some(false), args.with_filename); + + let args = parse_low_raw(["-I"]).unwrap(); + assert_eq!(Some(false), args.with_filename); + + let args = parse_low_raw(["-I", "-H"]).unwrap(); + assert_eq!(Some(true), args.with_filename); + + let args = parse_low_raw(["-H", "-I"]).unwrap(); + assert_eq!(Some(false), args.with_filename); +} + +/// -w/--word-regexp +#[derive(Debug)] +struct WordRegexp; + +impl Flag for WordRegexp { + fn is_switch(&self) -> bool { + true + } + fn name_short(&self) -> Option { + Some(b'w') + } + fn name_long(&self) -> &'static str { + "word-regexp" + } + fn doc_category(&self) -> Category { + Category::Search + } + fn doc_short(&self) -> &'static str { + r"Show matches surrounded by word boundaries." + } + fn doc_long(&self) -> &'static str { + r" +When enabled, ripgrep will only show matches surrounded by word boundaries. +This is equivalent to surrounding every pattern with \fB\\b{start-half}\fP and +\fB\\b{end-half}\fP. These are a custom syntax from ripgrep's default regex +engine that, unlike \fB\\b\fP, doesn't require matching a word character on one +side. That is, \fB\\b{start-half}\fP corresponds to matching \fB\\W|\\A\fP on +the left and \fB\\b{end-half}\fP corresponds to matching \fB\\W|\\z\fP on the +right. +.sp +This overrides the \flag{line-regexp} flag. +" + } + + fn update(&self, v: FlagValue, args: &mut LowArgs) -> anyhow::Result<()> { + assert!(v.unwrap_switch(), "--word-regexp has no negation"); + args.boundary = Some(BoundaryMode::Word); + Ok(()) + } +} + +#[cfg(test)] +#[test] +fn test_word_regexp() { + let args = parse_low_raw(None::<&str>).unwrap(); + assert_eq!(None, args.boundary); + + let args = parse_low_raw(["--word-regexp"]).unwrap(); + assert_eq!(Some(BoundaryMode::Word), args.boundary); + + let args = parse_low_raw(["-w"]).unwrap(); + assert_eq!(Some(BoundaryMode::Word), args.boundary); + + let args = parse_low_raw(["-x", "-w"]).unwrap(); + assert_eq!(Some(BoundaryMode::Word), args.boundary); + + let args = parse_low_raw(["-w", "-x"]).unwrap(); + assert_eq!(Some(BoundaryMode::Line), args.boundary); +} + +mod convert { + use std::ffi::{OsStr, OsString}; + + use anyhow::Context; + + pub(super) fn str(v: &OsStr) -> anyhow::Result<&str> { + let Some(s) = v.to_str() else { + anyhow::bail!("value is not valid UTF-8") + }; + Ok(s) + } + + pub(super) fn string(v: OsString) -> anyhow::Result { + let Ok(s) = v.into_string() else { + anyhow::bail!("value is not valid UTF-8") + }; + Ok(s) + } + + pub(super) fn usize(v: &OsStr) -> anyhow::Result { + str(v)?.parse().context("value is not a valid number") + } + + pub(super) fn u64(v: &OsStr) -> anyhow::Result { + str(v)?.parse().context("value is not a valid number") + } + + pub(super) fn human_readable_u64(v: &OsStr) -> anyhow::Result { + grep::cli::parse_human_readable_size(str(v)?).context("invalid size") + } + + pub(super) fn human_readable_usize(v: &OsStr) -> anyhow::Result { + let size = human_readable_u64(v)?; + let Ok(size) = usize::try_from(size) else { + anyhow::bail!("size is too big") + }; + Ok(size) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn available_shorts() { + let mut total = vec![false; 128]; + for byte in 0..=0x7F { + match byte { + b'.' | b'0'..=b'9' | b'A'..=b'Z' | b'a'..=b'z' => { + total[usize::from(byte)] = true + } + _ => continue, + } + } + + let mut taken = vec![false; 128]; + for flag in FLAGS.iter() { + let Some(short) = flag.name_short() else { continue }; + taken[usize::from(short)] = true; + } + + for byte in 0..=0x7F { + if total[usize::from(byte)] && !taken[usize::from(byte)] { + eprintln!("{}", char::from(byte)); + } + } + } + + #[test] + fn shorts_all_ascii_alphanumeric() { + for flag in FLAGS.iter() { + let Some(byte) = flag.name_short() else { continue }; + let long = flag.name_long(); + assert!( + byte.is_ascii_alphanumeric() || byte == b'.', + "\\x{byte:0X} is not a valid short flag for {long}", + ) + } + } + + #[test] + fn longs_all_ascii_alphanumeric() { + for flag in FLAGS.iter() { + let long = flag.name_long(); + let count = long.chars().count(); + assert!(count >= 2, "flag '{long}' is less than 2 characters"); + assert!( + long.chars().all(|c| c.is_ascii_alphanumeric() || c == '-'), + "flag '{long}' does not match ^[-0-9A-Za-z]+$", + ); + for alias in flag.aliases() { + let count = alias.chars().count(); + assert!( + count >= 2, + "flag '{long}' has alias '{alias}' that is \ + less than 2 characters", + ); + assert!( + alias + .chars() + .all(|c| c.is_ascii_alphanumeric() || c == '-'), + "flag '{long}' has alias '{alias}' that does not \ + match ^[-0-9A-Za-z]+$", + ); + } + let Some(negated) = flag.name_negated() else { continue }; + let count = negated.chars().count(); + assert!( + count >= 2, + "flag '{long}' has negation '{negated}' that is \ + less than 2 characters", + ); + assert!( + negated.chars().all(|c| c.is_ascii_alphanumeric() || c == '-'), + "flag '{long}' has negation '{negated}' that \ + does not match ^[-0-9A-Za-z]+$", + ); + } + } + + #[test] + fn shorts_no_duplicates() { + let mut taken = vec![false; 128]; + for flag in FLAGS.iter() { + let Some(short) = flag.name_short() else { continue }; + let long = flag.name_long(); + assert!( + !taken[usize::from(short)], + "flag {long} has duplicate short flag {}", + char::from(short) + ); + taken[usize::from(short)] = true; + } + } + + #[test] + fn longs_no_duplicates() { + use std::collections::BTreeSet; + + let mut taken = BTreeSet::new(); + for flag in FLAGS.iter() { + let long = flag.name_long(); + assert!(taken.insert(long), "flag {long} has a duplicate name"); + for alias in flag.aliases() { + assert!( + taken.insert(alias), + "flag {long} has an alias {alias} that is duplicative" + ); + } + let Some(negated) = flag.name_negated() else { continue }; + assert!( + taken.insert(negated), + "negated flag {negated} has a duplicate name" + ); + } + } + + #[test] + fn non_switches_have_variable_names() { + for flag in FLAGS.iter() { + if flag.is_switch() { + continue; + } + let long = flag.name_long(); + assert!( + flag.doc_variable().is_some(), + "flag '{long}' should have a variable name" + ); + } + } + + #[test] + fn switches_have_no_choices() { + for flag in FLAGS.iter() { + if !flag.is_switch() { + continue; + } + let long = flag.name_long(); + let choices = flag.doc_choices(); + assert!( + choices.is_empty(), + "switch flag '{long}' \ + should not have any choices but has some: {choices:?}", + ); + } + } + + #[test] + fn choices_ascii_alphanumeric() { + for flag in FLAGS.iter() { + let long = flag.name_long(); + for choice in flag.doc_choices() { + assert!( + choice.chars().all(|c| c.is_ascii_alphanumeric() + || c == '-' + || c == ':' + || c == '+'), + "choice '{choice}' for flag '{long}' does not match \ + ^[-+:0-9A-Za-z]+$", + ) + } + } + } +} diff --git a/userland/ripgrep/crates/core/flags/doc/help.rs b/userland/ripgrep/crates/core/flags/doc/help.rs new file mode 100644 index 0000000..c579bd2 --- /dev/null +++ b/userland/ripgrep/crates/core/flags/doc/help.rs @@ -0,0 +1,259 @@ +/*! +Provides routines for generating ripgrep's "short" and "long" help +documentation. + +The short version is used when the `-h` flag is given, while the long version +is used when the `--help` flag is given. +*/ + +use std::{collections::BTreeMap, fmt::Write}; + +use crate::flags::{Category, Flag, defs::FLAGS, doc::version}; + +const TEMPLATE_SHORT: &'static str = include_str!("template.short.help"); +const TEMPLATE_LONG: &'static str = include_str!("template.long.help"); + +/// Wraps `std::write!` and asserts there is no failure. +/// +/// We only write to `String` in this module. +macro_rules! write { + ($($tt:tt)*) => { std::write!($($tt)*).unwrap(); } +} + +/// Generate short documentation, i.e., for `-h`. +pub(crate) fn generate_short() -> String { + let mut cats: BTreeMap, Vec)> = + BTreeMap::new(); + let (mut maxcol1, mut maxcol2) = (0, 0); + for flag in FLAGS.iter().copied() { + let columns = + cats.entry(flag.doc_category()).or_insert((vec![], vec![])); + let (col1, col2) = generate_short_flag(flag); + maxcol1 = maxcol1.max(col1.len()); + maxcol2 = maxcol2.max(col2.len()); + columns.0.push(col1); + columns.1.push(col2); + } + let mut out = + TEMPLATE_SHORT.replace("!!VERSION!!", &version::generate_digits()); + for (cat, (col1, col2)) in cats.iter() { + let var = format!("!!{name}!!", name = cat.as_str()); + let val = format_short_columns(col1, col2, maxcol1, maxcol2); + out = out.replace(&var, &val); + } + out +} + +/// Generate short for a single flag. +/// +/// The first element corresponds to the flag name while the second element +/// corresponds to the documentation string. +fn generate_short_flag(flag: &dyn Flag) -> (String, String) { + let (mut col1, mut col2) = (String::new(), String::new()); + + // Some of the variable names are fine for longer form + // docs, but they make the succinct short help very noisy. + // So just shorten some of them. + let var = flag.doc_variable().map(|s| { + let mut s = s.to_string(); + s = s.replace("SEPARATOR", "SEP"); + s = s.replace("REPLACEMENT", "TEXT"); + s = s.replace("NUM+SUFFIX?", "NUM"); + s + }); + + // Generate the first column, the flag name. + if let Some(byte) = flag.name_short() { + let name = char::from(byte); + write!(col1, r"-{name}"); + write!(col1, r", "); + } + write!(col1, r"--{name}", name = flag.name_long()); + if let Some(var) = var.as_ref() { + write!(col1, r"={var}"); + } + + // And now the second column, with the description. + write!(col2, "{}", flag.doc_short()); + + (col1, col2) +} + +/// Write two columns of documentation. +/// +/// `maxcol1` should be the maximum length (in bytes) of the first column, +/// while `maxcol2` should be the maximum length (in bytes) of the second +/// column. +fn format_short_columns( + col1: &[String], + col2: &[String], + maxcol1: usize, + _maxcol2: usize, +) -> String { + assert_eq!(col1.len(), col2.len(), "columns must have equal length"); + const PAD: usize = 2; + let mut out = String::new(); + for (i, (c1, c2)) in col1.iter().zip(col2.iter()).enumerate() { + if i > 0 { + write!(out, "\n"); + } + + let pad = maxcol1 - c1.len() + PAD; + write!(out, " "); + write!(out, "{c1}"); + write!(out, "{}", " ".repeat(pad)); + write!(out, "{c2}"); + } + out +} + +/// Generate long documentation, i.e., for `--help`. +pub(crate) fn generate_long() -> String { + let mut cats = BTreeMap::new(); + for flag in FLAGS.iter().copied() { + let mut cat = cats.entry(flag.doc_category()).or_insert(String::new()); + if !cat.is_empty() { + write!(cat, "\n\n"); + } + generate_long_flag(flag, &mut cat); + } + + let mut out = + TEMPLATE_LONG.replace("!!VERSION!!", &version::generate_digits()); + for (cat, value) in cats.iter() { + let var = format!("!!{name}!!", name = cat.as_str()); + out = out.replace(&var, value); + } + out +} + +/// Write generated documentation for `flag` to `out`. +fn generate_long_flag(flag: &dyn Flag, out: &mut String) { + if let Some(byte) = flag.name_short() { + let name = char::from(byte); + write!(out, r" -{name}"); + if let Some(var) = flag.doc_variable() { + write!(out, r" {var}"); + } + write!(out, r", "); + } else { + write!(out, r" "); + } + + let name = flag.name_long(); + write!(out, r"--{name}"); + if let Some(var) = flag.doc_variable() { + write!(out, r"={var}"); + } + write!(out, "\n"); + + let doc = flag.doc_long().trim(); + let doc = super::render_custom_markup(doc, "flag", |name, out| { + let Some(flag) = crate::flags::parse::lookup(name) else { + unreachable!(r"found unrecognized \flag{{{name}}} in --help docs") + }; + if let Some(name) = flag.name_short() { + write!(out, r"-{}/", char::from(name)); + } + write!(out, r"--{}", flag.name_long()); + }); + let doc = super::render_custom_markup(&doc, "flag-negate", |name, out| { + let Some(flag) = crate::flags::parse::lookup(name) else { + unreachable!( + r"found unrecognized \flag-negate{{{name}}} in --help docs" + ) + }; + let Some(name) = flag.name_negated() else { + let long = flag.name_long(); + unreachable!( + "found \\flag-negate{{{long}}} in --help docs but \ + {long} does not have a negation" + ); + }; + write!(out, r"--{name}"); + }); + + let mut cleaned = remove_roff(&doc); + if let Some(negated) = flag.name_negated() { + // Flags that can be negated that aren't switches, like + // --context-separator, are somewhat weird. Because of that, the docs + // for those flags should discuss the semantics of negation explicitly. + // But for switches, the behavior is always the same. + if flag.is_switch() { + write!(cleaned, "\n\nThis flag can be disabled with --{negated}."); + } + } + let indent = " ".repeat(8); + let wrapopts = textwrap::Options::new(71) + // Normally I'd be fine with breaking at hyphens, but ripgrep's docs + // includes a lot of flag names, and they in turn contain hyphens. + // Breaking flag names across lines is not great. + .word_splitter(textwrap::WordSplitter::NoHyphenation); + for (i, paragraph) in cleaned.split("\n\n").enumerate() { + if i > 0 { + write!(out, "\n\n"); + } + let mut new = paragraph.to_string(); + if paragraph.lines().all(|line| line.starts_with(" ")) { + // Re-indent but don't refill so as to preserve line breaks + // in code/shell example snippets. + new = textwrap::indent(&new, &indent); + } else { + new = new.replace("\n", " "); + new = textwrap::refill(&new, &wrapopts); + new = textwrap::indent(&new, &indent); + } + write!(out, "{}", new.trim_end()); + } +} + +/// Removes roff syntax from `v` such that the result is approximately plain +/// text readable. +/// +/// This is basically a mish mash of heuristics based on the specific roff used +/// in the docs for the flags in this tool. If new kinds of roff are used in +/// the docs, then this may need to be updated to handle them. +fn remove_roff(v: &str) -> String { + let mut lines = vec![]; + for line in v.trim().lines() { + assert!(!line.is_empty(), "roff should have no empty lines"); + if line.starts_with(".") { + if line.starts_with(".IP ") { + let item_label = line + .split(" ") + .nth(1) + .expect("first argument to .IP") + .replace(r"\(bu", r"•") + .replace(r"\fB", "") + .replace(r"\fP", ":"); + lines.push(format!("{item_label}")); + } else if line.starts_with(".IB ") || line.starts_with(".BI ") { + let pieces = line + .split_whitespace() + .skip(1) + .collect::>() + .concat(); + lines.push(format!("{pieces}")); + } else if line.starts_with(".sp") + || line.starts_with(".PP") + || line.starts_with(".TP") + { + lines.push("".to_string()); + } + } else if line.starts_with(r"\fB") && line.ends_with(r"\fP") { + let line = line.replace(r"\fB", "").replace(r"\fP", ""); + lines.push(format!("{line}:")); + } else { + lines.push(line.to_string()); + } + } + // Squash multiple adjacent paragraph breaks into one. + lines.dedup_by(|l1, l2| l1.is_empty() && l2.is_empty()); + lines + .join("\n") + .replace(r"\fB", "") + .replace(r"\fI", "") + .replace(r"\fP", "") + .replace(r"\-", "-") + .replace(r"\\", r"\") +} diff --git a/userland/ripgrep/crates/core/flags/doc/man.rs b/userland/ripgrep/crates/core/flags/doc/man.rs new file mode 100644 index 0000000..c89a6d3 --- /dev/null +++ b/userland/ripgrep/crates/core/flags/doc/man.rs @@ -0,0 +1,110 @@ +/*! +Provides routines for generating ripgrep's man page in `roff` format. +*/ + +use std::{collections::BTreeMap, fmt::Write}; + +use crate::flags::{Flag, defs::FLAGS, doc::version}; + +const TEMPLATE: &'static str = include_str!("template.rg.1"); + +/// Wraps `std::write!` and asserts there is no failure. +/// +/// We only write to `String` in this module. +macro_rules! write { + ($($tt:tt)*) => { std::write!($($tt)*).unwrap(); } +} + +/// Wraps `std::writeln!` and asserts there is no failure. +/// +/// We only write to `String` in this module. +macro_rules! writeln { + ($($tt:tt)*) => { std::writeln!($($tt)*).unwrap(); } +} + +/// Returns a `roff` formatted string corresponding to ripgrep's entire man +/// page. +pub(crate) fn generate() -> String { + let mut cats = BTreeMap::new(); + for flag in FLAGS.iter().copied() { + let mut cat = cats.entry(flag.doc_category()).or_insert(String::new()); + if !cat.is_empty() { + writeln!(cat, ".sp"); + } + generate_flag(flag, &mut cat); + } + + let mut out = TEMPLATE.replace("!!VERSION!!", &version::generate_digits()); + for (cat, value) in cats.iter() { + let var = format!("!!{name}!!", name = cat.as_str()); + out = out.replace(&var, value); + } + out +} + +/// Writes `roff` formatted documentation for `flag` to `out`. +fn generate_flag(flag: &'static dyn Flag, out: &mut String) { + if let Some(byte) = flag.name_short() { + let name = char::from(byte); + write!(out, r"\fB\-{name}\fP"); + if let Some(var) = flag.doc_variable() { + write!(out, r" \fI{var}\fP"); + } + write!(out, r", "); + } + + let name = flag.name_long().replace("-", r"\-"); + write!(out, r"\fB\-\-{name}\fP"); + if let Some(var) = flag.doc_variable() { + write!(out, r"=\fI{var}\fP"); + } + write!(out, "\n"); + + writeln!(out, ".RS 4"); + let doc = flag.doc_long().trim(); + // Convert \flag{foo} into something nicer. + let doc = super::render_custom_markup(doc, "flag", |name, out| { + let Some(flag) = crate::flags::parse::lookup(name) else { + unreachable!(r"found unrecognized \flag{{{name}}} in roff docs") + }; + out.push_str(r"\fB"); + if let Some(name) = flag.name_short() { + write!(out, r"\-{}/", char::from(name)); + } + write!(out, r"\-\-{}", flag.name_long().replace("-", r"\-")); + out.push_str(r"\fP"); + }); + // Convert \flag-negate{foo} into something nicer. + let doc = super::render_custom_markup(&doc, "flag-negate", |name, out| { + let Some(flag) = crate::flags::parse::lookup(name) else { + unreachable!( + r"found unrecognized \flag-negate{{{name}}} in roff docs" + ) + }; + let Some(name) = flag.name_negated() else { + let long = flag.name_long(); + unreachable!( + "found \\flag-negate{{{long}}} in roff docs but \ + {long} does not have a negation" + ); + }; + out.push_str(r"\fB"); + write!(out, r"\-\-{name}"); + out.push_str(r"\fP"); + }); + writeln!(out, "{doc}"); + if let Some(negated) = flag.name_negated() { + // Flags that can be negated that aren't switches, like + // --context-separator, are somewhat weird. Because of that, the docs + // for those flags should discuss the semantics of negation explicitly. + // But for switches, the behavior is always the same. + if flag.is_switch() { + writeln!(out, ".sp"); + writeln!( + out, + r"This flag can be disabled with \fB\-\-{negated}\fP." + ); + } + } + writeln!(out, ".RE"); +} diff --git a/userland/ripgrep/crates/core/flags/doc/mod.rs b/userland/ripgrep/crates/core/flags/doc/mod.rs new file mode 100644 index 0000000..c52a024 --- /dev/null +++ b/userland/ripgrep/crates/core/flags/doc/mod.rs @@ -0,0 +1,38 @@ +/*! +Modules for generating documentation for ripgrep's flags. +*/ + +pub(crate) mod help; +pub(crate) mod man; +pub(crate) mod version; + +/// Searches for `\tag{...}` occurrences in `doc` and calls `replacement` for +/// each such tag found. +/// +/// The first argument given to `replacement` is the tag value, `...`. The +/// second argument is the buffer that accumulates the full replacement text. +/// +/// Since this function is only intended to be used on doc strings written into +/// the program source code, callers should panic in `replacement` if there are +/// any errors or unexpected circumstances. +fn render_custom_markup( + mut doc: &str, + tag: &str, + mut replacement: impl FnMut(&str, &mut String), +) -> String { + let mut out = String::with_capacity(doc.len()); + let tag_prefix = format!(r"\{tag}{{"); + while let Some(offset) = doc.find(&tag_prefix) { + out.push_str(&doc[..offset]); + + let start = offset + tag_prefix.len(); + let Some(end) = doc[start..].find('}').map(|i| start + i) else { + unreachable!(r"found {tag_prefix} without closing }}"); + }; + let name = &doc[start..end]; + replacement(name, &mut out); + doc = &doc[end + 1..]; + } + out.push_str(doc); + out +} diff --git a/userland/ripgrep/crates/core/flags/doc/template.long.help b/userland/ripgrep/crates/core/flags/doc/template.long.help new file mode 100644 index 0000000..83c20c1 --- /dev/null +++ b/userland/ripgrep/crates/core/flags/doc/template.long.help @@ -0,0 +1,61 @@ +ripgrep !!VERSION!! +Andrew Gallant + +ripgrep (rg) recursively searches the current directory for lines matching +a regex pattern. By default, ripgrep will respect gitignore rules and +automatically skip hidden files/directories and binary files. + +Use -h for short descriptions and --help for more details. + +Project home page: https://github.com/BurntSushi/ripgrep + +USAGE: + rg [OPTIONS] PATTERN [PATH ...] + rg [OPTIONS] -e PATTERN ... [PATH ...] + rg [OPTIONS] -f PATTERNFILE ... [PATH ...] + rg [OPTIONS] --files [PATH ...] + rg [OPTIONS] --type-list + command | rg [OPTIONS] PATTERN + rg [OPTIONS] --help + rg [OPTIONS] --version + +POSITIONAL ARGUMENTS: + + A regular expression used for searching. To match a pattern beginning + with a dash, use the -e/--regexp flag. + + For example, to search for the literal '-foo', you can use this flag: + + rg -e -foo + + You can also use the special '--' delimiter to indicate that no more + flags will be provided. Namely, the following is equivalent to the + above: + + rg -- -foo + + ... + A file or directory to search. Directories are searched recursively. + File paths specified on the command line override glob and ignore + rules. + +INPUT OPTIONS: +!!input!! + +SEARCH OPTIONS: +!!search!! + +FILTER OPTIONS: +!!filter!! + +OUTPUT OPTIONS: +!!output!! + +OUTPUT MODES: +!!output-modes!! + +LOGGING OPTIONS: +!!logging!! + +OTHER BEHAVIORS: +!!other-behaviors!! diff --git a/userland/ripgrep/crates/core/flags/doc/template.rg.1 b/userland/ripgrep/crates/core/flags/doc/template.rg.1 new file mode 100644 index 0000000..ac3e607 --- /dev/null +++ b/userland/ripgrep/crates/core/flags/doc/template.rg.1 @@ -0,0 +1,424 @@ +.TH RG 1 2026-07-15 "!!VERSION!!" "User Commands" +. +. +.SH NAME +rg \- recursively search the current directory for lines matching a pattern +. +. +.SH SYNOPSIS +.\" I considered using GNU troff's .SY and .YS "synopsis" macros here, but it +.\" looks like they aren't portable. Specifically, they don't appear to be in +.\" BSD's mdoc used on macOS. +.sp +\fBrg\fP [\fIOPTIONS\fP] \fIPATTERN\fP [\fIPATH\fP...] +.sp +\fBrg\fP [\fIOPTIONS\fP] \fB\-e\fP \fIPATTERN\fP... [\fIPATH\fP...] +.sp +\fBrg\fP [\fIOPTIONS\fP] \fB\-f\fP \fIPATTERNFILE\fP... [\fIPATH\fP...] +.sp +\fBrg\fP [\fIOPTIONS\fP] \fB\-\-files\fP [\fIPATH\fP...] +.sp +\fBrg\fP [\fIOPTIONS\fP] \fB\-\-type\-list\fP +.sp +\fIcommand\fP | \fBrg\fP [\fIOPTIONS\fP] \fIPATTERN\fP +.sp +\fBrg\fP [\fIOPTIONS\fP] \fB\-\-help\fP +.sp +\fBrg\fP [\fIOPTIONS\fP] \fB\-\-version\fP +. +. +.SH DESCRIPTION +ripgrep (rg) recursively searches the current directory for a regex pattern. +By default, ripgrep will respect your \fB.gitignore\fP and automatically skip +hidden files/directories and binary files. +.sp +ripgrep's default regex engine uses finite automata and guarantees linear +time searching. Because of this, features like backreferences and arbitrary +look-around are not supported. However, if ripgrep is built with PCRE2, +then the \fB\-P/\-\-pcre2\fP flag can be used to enable backreferences and +look-around. +.sp +ripgrep supports configuration files. Set \fBRIPGREP_CONFIG_PATH\fP to a +configuration file. The file can specify one shell argument per line. Lines +starting with \fB#\fP are ignored. For more details, see \fBCONFIGURATION +FILES\fP below. +.sp +ripgrep will automatically detect if stdin is a readable file and search stdin +for a regex pattern, e.g. \fBls | rg foo\fP. In some environments, stdin may +exist when it shouldn't. To turn off stdin detection, one can explicitly +specify the directory to search, e.g. \fBrg foo ./\fP. +.sp +Like other tools such as \fBls\fP, ripgrep will alter its output depending on +whether stdout is connected to a tty. By default, when printing a tty, ripgrep +will enable colors, line numbers and a heading format that lists each matching +file path once instead of once per matching line. +.sp +Tip: to disable all smart filtering and make ripgrep behave a bit more like +classical grep, use \fBrg -uuu\fP. +. +. +.SH REGEX SYNTAX +ripgrep uses Rust's regex engine by default, which documents its syntax: +\fIhttps://docs.rs/regex/1.*/regex/#syntax\fP +.sp +ripgrep uses byte-oriented regexes, which has some additional documentation: +\fIhttps://docs.rs/regex/1.*/regex/bytes/index.html#syntax\fP +.sp +To a first approximation, ripgrep uses Perl-like regexes without look-around or +backreferences. This makes them very similar to the "extended" (ERE) regular +expressions supported by *egrep*, but with a few additional features like +Unicode character classes. +.sp +If you're using ripgrep with the \fB\-P/\-\-pcre2\fP flag, then please consult +\fIhttps://www.pcre.org\fP or the PCRE2 man pages for documentation on the +supported syntax. +. +. +.SH POSITIONAL ARGUMENTS +.TP 12 +\fIPATTERN\fP +A regular expression used for searching. To match a pattern beginning with a +dash, use the \fB\-e/\-\-regexp\fP option. +.TP 12 +\fIPATH\fP +A file or directory to search. Directories are searched recursively. File paths +specified explicitly on the command line override glob and ignore rules. +. +. +.SH OPTIONS +This section documents all flags that ripgrep accepts. Flags are grouped into +categories below according to their function. +.sp +Note that many options can be turned on and off. In some cases, those flags are +not listed explicitly below. For example, the \fB\-\-column\fP flag (listed +below) enables column numbers in ripgrep's output, but the \fB\-\-no\-column\fP +flag (not listed below) disables them. The reverse can also exist. For example, +the \fB\-\-no\-ignore\fP flag (listed below) disables ripgrep's \fBgitignore\fP +logic, but the \fB\-\-ignore\fP flag (not listed below) enables it. These +flags are useful for overriding a ripgrep configuration file (or alias) on the +command line. Each flag's documentation notes whether an inverted flag exists. +In all cases, the flag specified last takes precedence. +. +.SS INPUT OPTIONS +!!input!! +. +.SS SEARCH OPTIONS +!!search!! +. +.SS FILTER OPTIONS +!!filter!! +. +.SS OUTPUT OPTIONS +!!output!! +. +.SS OUTPUT MODES +!!output-modes!! +. +.SS LOGGING OPTIONS +!!logging!! +. +.SS OTHER BEHAVIORS +!!other-behaviors!! +. +. +.SH EXIT STATUS +If ripgrep finds a match, then the exit status of the program is \fB0\fP. +If no match could be found, then the exit status is \fB1\fP. If an error +occurred, then the exit status is always \fB2\fP unless ripgrep was run with +the \fB\-q/\-\-quiet\fP flag and a match was found. In summary: +.sp +.IP \(bu 3n +\fB0\fP exit status occurs only when at least one match was found, and if +no error occurred, unless \fB\-q/\-\-quiet\fP was given. +. +.IP \(bu 3n +\fB1\fP exit status occurs only when no match was found and no error occurred. +. +.IP \(bu 3n +\fB2\fP exit status occurs when an error occurred. This is true for both +catastrophic errors (e.g., a regex syntax error) and for soft errors (e.g., +unable to read a file). +. +. +.SH AUTOMATIC FILTERING +ripgrep does a fair bit of automatic filtering by default. This section +describes that filtering and how to control it. +.sp +\fBTIP\fP: To disable automatic filtering, use \fBrg -uuu\fP. +.sp +ripgrep's automatic "smart" filtering is one of the most apparent +differentiating features between ripgrep and other tools like \fBgrep\fP. As +such, its behavior may be surprising to users that aren't expecting it. +.sp +ripgrep does four types of filtering automatically: +.sp +. +.IP 1. 3n +Files and directories that match ignore rules are not searched. +.IP 2. 3n +Hidden files and directories are not searched. +.IP 3. 3n +Binary files (files with a \fBNUL\fP byte) are not searched. +.IP 4. 3n +Symbolic links are not followed. +.PP +The first type of filtering is the most sophisticated. ripgrep will attempt to +respect your \fBgitignore\fP rules as faithfully as possible. In particular, +this includes the following: +. +.IP \(bu 3n +Any global rules, e.g., in \fB$HOME/.config/git/ignore\fP. +. +.IP \(bu 3n +Any rules in relevant \fB.gitignore\fP files. This includes \fB.gitignore\fP +files in parent directories that are part of the same \fBgit\fP repository. +(Unless \fB\-\-no\-require\-git\fP is given.) +. +.IP \(bu 3n +Any local rules, e.g., in \fB.git/info/exclude\fP. +.PP +In some cases, ripgrep and \fBgit\fP will not always be in sync in terms +of which files are ignored. For example, a file that is ignored via +\fB.gitignore\fP but is tracked by \fBgit\fP would not be searched by ripgrep +even though \fBgit\fP tracks it. This is unlikely to ever be fixed. Instead, +you should either make sure your exclude rules match the files you track +precisely, or otherwise use \fBgit grep\fP for search. +.sp +Additional ignore rules can be provided outside of a \fBgit\fP context: +. +.IP \(bu 3n +Any rules in \fB.ignore\fP. ripgrep will also respect \fB.ignore\fP files in +parent directories. +. +.IP \(bu 3n +Any rules in \fB.rgignore\fP. ripgrep will also respect \fB.rgignore\fP files +in parent directories. +. +.IP \(bu 3n +Any rules in files specified with the \fB\-\-ignore\-file\fP flag. +.PP +The precedence of ignore rules is as follows, with later items overriding +earlier items: +. +.IP \(bu 3n +Files given by \fB\-\-ignore\-file\fP. +. +.IP \(bu 3n +Global gitignore rules, e.g., from \fB$HOME/.config/git/ignore\fP. +. +.IP \(bu 3n +Local rules from \fB.git/info/exclude\fP. +. +.IP \(bu 3n +Rules from \fB.gitignore\fP. +. +.IP \(bu 3n +Rules from \fB.ignore\fP. +. +.IP \(bu 3n +Rules from \fB.rgignore\fP. +.PP +So for example, if \fIfoo\fP were in a \fB.gitignore\fP and \fB!\fP\fIfoo\fP +were in an \fB.rgignore\fP, then \fIfoo\fP would not be ignored since +\fB.rgignore\fP takes precedence over \fB.gitignore\fP. +.sp +Each of the types of filtering can be configured via command line flags: +. +.IP \(bu 3n +There are several flags starting with \fB\-\-no\-ignore\fP that toggle which, +if any, ignore rules are respected. \fB\-\-no\-ignore\fP by itself will disable +all +of them. +. +.IP \(bu 3n +\fB\-./\-\-hidden\fP will force ripgrep to search hidden files and directories. +. +.IP \(bu 3n +\fB\-\-binary\fP will force ripgrep to search binary files. +. +.IP \(bu 3n +\fB\-L/\-\-follow\fP will force ripgrep to follow symlinks. +.PP +As a special short hand, the \fB\-u\fP flag can be specified up to three times. +Each additional time incrementally decreases filtering: +. +.IP \(bu 3n +\fB\-u\fP is equivalent to \fB\-\-no\-ignore\fP. +. +.IP \(bu 3n +\fB\-uu\fP is equivalent to \fB\-\-no\-ignore \-\-hidden\fP. +. +.IP \(bu 3n +\fB\-uuu\fP is equivalent to \fB\-\-no\-ignore \-\-hidden \-\-binary\fP. +.PP +In particular, \fBrg -uuu\fP should search the same exact content as \fBgrep +-r\fP. +. +. +.SH CONFIGURATION FILES +ripgrep supports reading configuration files that change ripgrep's default +behavior. The format of the configuration file is an "rc" style and is very +simple. It is defined by two rules: +. +.IP 1. 3n +Every line is a shell argument, after trimming whitespace. +. +.IP 2. 3n +Lines starting with \fB#\fP (optionally preceded by any amount of whitespace) +are ignored. +.PP +ripgrep will look for a single configuration file if and only if the +\fBRIPGREP_CONFIG_PATH\fP environment variable is set and is non-empty. +ripgrep will parse arguments from this file on startup and will behave as if +the arguments in this file were prepended to any explicit arguments given to +ripgrep on the command line. Note though that the \fBrg\fP command you run +must still be valid. That is, it must always contain at least one pattern at +the command line, even if the configuration file uses the \fB\-e/\-\-regexp\fP +flag. +.sp +For example, if your ripgreprc file contained a single line: +.sp +.EX + \-\-smart\-case +.EE +.sp +then the following command +.sp +.EX + RIPGREP_CONFIG_PATH=wherever/.ripgreprc rg foo +.EE +.sp +would behave identically to the following command: +.sp +.EX + rg \-\-smart-case foo +.EE +.sp +Another example is adding types, like so: +.sp +.EX + \-\-type-add + web:*.{html,css,js}* +.EE +.sp +The above would behave identically to the following command: +.sp +.EX + rg \-\-type\-add 'web:*.{html,css,js}*' foo +.EE +.sp +The same applies to using globs. This: +.sp +.EX + \-\-glob=!.git +.EE +.sp +or this: +.sp +.EX + \-\-glob + !.git +.EE +.sp +would behave identically to the following command: +.sp +.EX + rg \-\-glob '!.git' foo +.EE +.sp +The bottom line is that every shell argument needs to be on its own line. So +for example, a config file containing +.sp +.EX + \-j 4 +.EE +.sp +is probably not doing what you intend. Instead, you want +.sp +.EX + \-j + 4 +.EE +.sp +or +.sp +.EX + \-j4 +.EE +.sp +ripgrep also provides a flag, \fB\-\-no\-config\fP, that when present will +suppress any and all support for configuration. This includes any future +support for auto-loading configuration files from pre-determined paths. +.sp +Conflicts between configuration files and explicit arguments are handled +exactly like conflicts in the same command line invocation. That is, assuming +your config file contains only \fB\-\-smart\-case\fP, then this command: +.sp +.EX + RIPGREP_CONFIG_PATH=wherever/.ripgreprc rg foo \-\-case\-sensitive +.EE +.sp +is exactly equivalent to +.sp +.EX + rg \-\-smart\-case foo \-\-case\-sensitive +.EE +.sp +in which case, the \fB\-\-case\-sensitive\fP flag would override the +\fB\-\-smart\-case\fP flag. +. +. +.SH SHELL COMPLETION +Shell completion files are included in the release tarball for Bash, Fish, Zsh +and PowerShell. +.sp +For \fBbash\fP, move \fBrg.bash\fP to \fB$XDG_CONFIG_HOME/bash_completion\fP or +\fB/etc/bash_completion.d/\fP. +.sp +For \fBfish\fP, move \fBrg.fish\fP to \fB$HOME/.config/fish/completions\fP. +.sp +For \fBzsh\fP, move \fB_rg\fP to one of your \fB$fpath\fP directories. +. +. +.SH CAVEATS +ripgrep may abort unexpectedly when using default settings if it searches a +file that is simultaneously truncated. This behavior can be avoided by passing +the \fB\-\-no\-mmap\fP flag which will forcefully disable the use of memory +maps in all cases. +.sp +ripgrep may use a large amount of memory depending on a few factors. Firstly, +if ripgrep uses parallelism for search (the default), then the entire +output for each individual file is buffered into memory in order to prevent +interleaving matches in the output. To avoid this, you can disable parallelism +with the \fB\-j1\fP flag. Secondly, ripgrep always needs to have at least a +single line in memory in order to execute a search. A file with a very long +line can thus cause ripgrep to use a lot of memory. Generally, this only occurs +when searching binary data with the \fB\-a/\-\-text\fP flag enabled. (When the +\fB\-a/\-\-text\fP flag isn't enabled, ripgrep will replace all NUL bytes with +line terminators, which typically prevents exorbitant memory usage.) Thirdly, +when ripgrep searches a large file using a memory map, the process will likely +report its resident memory usage as the size of the file. However, this does +not mean ripgrep actually needed to use that much heap memory; the operating +system will generally handle this for you. +. +. +.SH VERSION +!!VERSION!! +. +. +.SH HOMEPAGE +\fIhttps://github.com/BurntSushi/ripgrep\fP +.sp +Please report bugs and feature requests to the issue tracker. Please do your +best to provide a reproducible test case for bugs. This should include the +corpus being searched, the \fBrg\fP command, the actual output and the expected +output. Please also include the output of running the same \fBrg\fP command but +with the \fB\-\-debug\fP flag. +.sp +If you have questions that don't obviously fall into the "bug" or "feature +request" category, then they are welcome in the Discussions section of the +issue tracker: \fIhttps://github.com/BurntSushi/ripgrep/discussions\fP. +. +. +.SH AUTHORS +Andrew Gallant <\fIjamslam@gmail.com\fP> diff --git a/userland/ripgrep/crates/core/flags/doc/template.short.help b/userland/ripgrep/crates/core/flags/doc/template.short.help new file mode 100644 index 0000000..0a28566 --- /dev/null +++ b/userland/ripgrep/crates/core/flags/doc/template.short.help @@ -0,0 +1,38 @@ +ripgrep !!VERSION!! +Andrew Gallant + +ripgrep (rg) recursively searches the current directory for lines matching +a regex pattern. By default, ripgrep will respect gitignore rules and +automatically skip hidden files/directories and binary files. + +Use -h for short descriptions and --help for more details. + +Project home page: https://github.com/BurntSushi/ripgrep + +USAGE: + rg [OPTIONS] PATTERN [PATH ...] + +POSITIONAL ARGUMENTS: + A regular expression used for searching. + ... A file or directory to search. + +INPUT OPTIONS: +!!input!! + +SEARCH OPTIONS: +!!search!! + +FILTER OPTIONS: +!!filter!! + +OUTPUT OPTIONS: +!!output!! + +OUTPUT MODES: +!!output-modes!! + +LOGGING OPTIONS: +!!logging!! + +OTHER BEHAVIORS: +!!other-behaviors!! diff --git a/userland/ripgrep/crates/core/flags/doc/version.rs b/userland/ripgrep/crates/core/flags/doc/version.rs new file mode 100644 index 0000000..bf15622 --- /dev/null +++ b/userland/ripgrep/crates/core/flags/doc/version.rs @@ -0,0 +1,173 @@ +/*! +Provides routines for generating version strings. + +Version strings can be just the digits, an overall short one-line description +or something more verbose that includes things like CPU target feature support. +*/ + +use std::fmt::Write; + +/// Generates just the numerical part of the version of ripgrep. +/// +/// This includes the git revision hash. +pub(crate) fn generate_digits() -> String { + let semver = option_env!("CARGO_PKG_VERSION").unwrap_or("N/A"); + match option_env!("RIPGREP_BUILD_GIT_HASH") { + None => semver.to_string(), + Some(hash) => format!("{semver} (rev {hash})"), + } +} + +/// Generates a short version string of the form `ripgrep x.y.z`. +pub(crate) fn generate_short() -> String { + let digits = generate_digits(); + format!("ripgrep {digits}") +} + +/// Generates a longer multi-line version string. +/// +/// This includes not only the version of ripgrep but some other information +/// about its build. For example, SIMD support and PCRE2 support. +pub(crate) fn generate_long() -> String { + let (compile, runtime) = (compile_cpu_features(), runtime_cpu_features()); + + let mut out = String::new(); + writeln!(out, "{}", generate_short()).unwrap(); + writeln!(out).unwrap(); + writeln!(out, "features:{}", features().join(",")).unwrap(); + if !compile.is_empty() { + writeln!(out, "simd(compile):{}", compile.join(",")).unwrap(); + } + if !runtime.is_empty() { + writeln!(out, "simd(runtime):{}", runtime.join(",")).unwrap(); + } + let (pcre2_version, _) = generate_pcre2(); + writeln!(out, "\n{pcre2_version}").unwrap(); + out +} + +/// Generates multi-line version string with PCRE2 information. +/// +/// This also returns whether PCRE2 is actually available in this build of +/// ripgrep. +pub(crate) fn generate_pcre2() -> (String, bool) { + let mut out = String::new(); + + #[cfg(feature = "pcre2")] + { + use grep::pcre2; + + let (major, minor) = pcre2::version(); + write!(out, "PCRE2 {}.{} is available", major, minor).unwrap(); + if cfg!(target_pointer_width = "64") && pcre2::is_jit_available() { + writeln!(out, " (JIT is available)").unwrap(); + } else { + writeln!(out, " (JIT is unavailable)").unwrap(); + } + (out, true) + } + + #[cfg(not(feature = "pcre2"))] + { + writeln!(out, "PCRE2 is not available in this build of ripgrep.") + .unwrap(); + (out, false) + } +} + +/// Returns the relevant SIMD features supported by the CPU at runtime. +/// +/// This is kind of a dirty violation of abstraction, since it assumes +/// knowledge about what specific SIMD features are being used by various +/// components. +fn runtime_cpu_features() -> Vec { + #[cfg(target_arch = "x86_64")] + { + let mut features = vec![]; + + let sse2 = is_x86_feature_detected!("sse2"); + features.push(format!("{sign}SSE2", sign = sign(sse2))); + + let ssse3 = is_x86_feature_detected!("ssse3"); + features.push(format!("{sign}SSSE3", sign = sign(ssse3))); + + let avx2 = is_x86_feature_detected!("avx2"); + features.push(format!("{sign}AVX2", sign = sign(avx2))); + + features + } + #[cfg(target_arch = "aarch64")] + { + let mut features = vec![]; + + // memchr and aho-corasick only use NEON when it is available at + // compile time. This isn't strictly necessary, but NEON is supposed + // to be available for all aarch64 targets. If this isn't true, please + // file an issue at https://github.com/BurntSushi/memchr. + let neon = cfg!(target_feature = "neon"); + features.push(format!("{sign}NEON", sign = sign(neon))); + + features + } + #[cfg(not(any(target_arch = "x86_64", target_arch = "aarch64")))] + { + vec![] + } +} + +/// Returns the SIMD features supported while compiling ripgrep. +/// +/// In essence, any features listed here are required to run ripgrep correctly. +/// +/// This is kind of a dirty violation of abstraction, since it assumes +/// knowledge about what specific SIMD features are being used by various +/// components. +/// +/// An easy way to enable everything available on your current CPU is to +/// compile ripgrep with `RUSTFLAGS="-C target-cpu=native"`. But note that +/// the binary produced by this will not be portable. +fn compile_cpu_features() -> Vec { + #[cfg(target_arch = "x86_64")] + { + let mut features = vec![]; + + let sse2 = cfg!(target_feature = "sse2"); + features.push(format!("{sign}SSE2", sign = sign(sse2))); + + let ssse3 = cfg!(target_feature = "ssse3"); + features.push(format!("{sign}SSSE3", sign = sign(ssse3))); + + let avx2 = cfg!(target_feature = "avx2"); + features.push(format!("{sign}AVX2", sign = sign(avx2))); + + features + } + #[cfg(target_arch = "aarch64")] + { + let mut features = vec![]; + + let neon = cfg!(target_feature = "neon"); + features.push(format!("{sign}NEON", sign = sign(neon))); + + features + } + #[cfg(not(any(target_arch = "x86_64", target_arch = "aarch64")))] + { + vec![] + } +} + +/// Returns a list of "features" supported (or not) by this build of ripgrpe. +fn features() -> Vec { + let mut features = vec![]; + + let pcre2 = cfg!(feature = "pcre2"); + features.push(format!("{sign}pcre2", sign = sign(pcre2))); + + features +} + +/// Returns `+` when `enabled` is `true` and `-` otherwise. +fn sign(enabled: bool) -> &'static str { + if enabled { "+" } else { "-" } +} diff --git a/userland/ripgrep/crates/core/flags/hiargs.rs b/userland/ripgrep/crates/core/flags/hiargs.rs new file mode 100644 index 0000000..33430de --- /dev/null +++ b/userland/ripgrep/crates/core/flags/hiargs.rs @@ -0,0 +1,1480 @@ +/*! +Provides the definition of high level arguments from CLI flags. +*/ + +use std::{ + collections::HashSet, + path::{Path, PathBuf}, +}; + +use { + bstr::BString, + grep::printer::{ColorSpecs, SummaryKind}, +}; + +use crate::{ + flags::lowargs::{ + BinaryMode, BoundaryMode, BufferMode, CaseMode, ColorChoice, + ContextMode, ContextSeparator, EncodingMode, EngineChoice, + FieldContextSeparator, FieldMatchSeparator, LowArgs, MmapMode, Mode, + PatternSource, SearchMode, SortMode, SortModeKind, TypeChange, + }, + haystack::{Haystack, HaystackBuilder}, + search::{PatternMatcher, Printer, SearchWorker, SearchWorkerBuilder}, +}; + +/// A high level representation of CLI arguments. +/// +/// The distinction between low and high level arguments is somewhat arbitrary +/// and wishy washy. The main idea here is that high level arguments generally +/// require all of CLI parsing to be finished. For example, one cannot +/// construct a glob matcher until all of the glob patterns are known. +/// +/// So while low level arguments are collected during parsing itself, high +/// level arguments aren't created until parsing has completely finished. +#[derive(Debug)] +pub(crate) struct HiArgs { + binary: BinaryDetection, + boundary: Option, + buffer: BufferMode, + byte_offset: bool, + case: CaseMode, + color: ColorChoice, + colors: grep::printer::ColorSpecs, + column: bool, + context: ContextMode, + context_separator: ContextSeparator, + crlf: bool, + cwd: PathBuf, + dfa_size_limit: Option, + encoding: EncodingMode, + engine: EngineChoice, + field_context_separator: FieldContextSeparator, + field_match_separator: FieldMatchSeparator, + file_separator: Option>, + fixed_strings: bool, + follow: bool, + globs: ignore::overrides::Override, + heading: bool, + hidden: bool, + hyperlink_config: grep::printer::HyperlinkConfig, + ignore_file_case_insensitive: bool, + ignore_file: Vec, + include_zero: bool, + invert_match: bool, + is_terminal_stdout: bool, + line_number: bool, + max_columns: Option, + max_columns_preview: bool, + max_count: Option, + max_depth: Option, + max_filesize: Option, + mmap_choice: grep::searcher::MmapChoice, + mode: Mode, + multiline: bool, + multiline_dotall: bool, + no_ignore_dot: bool, + no_ignore_exclude: bool, + no_ignore_files: bool, + no_ignore_global: bool, + no_ignore_parent: bool, + no_ignore_vcs: bool, + no_require_git: bool, + no_unicode: bool, + null_data: bool, + one_file_system: bool, + only_matching: bool, + path_separator: Option, + paths: Paths, + path_terminator: Option, + patterns: Patterns, + pre: Option, + pre_globs: ignore::overrides::Override, + quiet: bool, + quit_after_match: bool, + regex_size_limit: Option, + replace: Option, + search_zip: bool, + sort: Option, + stats: Option, + stop_on_nonmatch: bool, + threads: usize, + trim: bool, + types: ignore::types::Types, + vimgrep: bool, + with_filename: bool, +} + +impl HiArgs { + /// Convert low level arguments into high level arguments. + /// + /// This process can fail for a variety of reasons. For example, invalid + /// globs or some kind of environment issue. + pub(crate) fn from_low_args(mut low: LowArgs) -> anyhow::Result { + // Callers should not be trying to convert low-level arguments when + // a short-circuiting special mode is present. + assert_eq!(None, low.special, "special mode demands short-circuiting"); + // If the sorting mode isn't supported, then we bail loudly. I'm not + // sure if this is the right thing to do. We could silently "not sort" + // as well. If we wanted to go that route, then we could just set + // `low.sort = None` if `supported()` returns an error. + if let Some(ref sort) = low.sort { + sort.supported()?; + } + + // We modify the mode in-place on `low` so that subsequent conversions + // see the correct mode. + match low.mode { + Mode::Search(ref mut mode) => match *mode { + // treat `-v --count-matches` as `-v --count` + SearchMode::CountMatches if low.invert_match => { + *mode = SearchMode::Count; + } + // treat `-o --count` as `--count-matches` + SearchMode::Count if low.only_matching => { + *mode = SearchMode::CountMatches; + } + _ => {} + }, + _ => {} + } + + let mut state = State::new()?; + let patterns = Patterns::from_low_args(&mut state, &mut low)?; + let paths = Paths::from_low_args(&mut state, &patterns, &mut low)?; + + let binary = BinaryDetection::from_low_args(&state, &low); + let colors = take_color_specs(&mut state, &mut low); + let hyperlink_config = take_hyperlink_config(&mut state, &mut low)?; + let stats = stats(&low); + let types = types(&low)?; + let globs = globs(&state, &low)?; + let pre_globs = preprocessor_globs(&state, &low)?; + + let color = match low.color { + ColorChoice::Auto if !state.is_terminal_stdout => { + ColorChoice::Never + } + _ => low.color, + }; + let column = low.column.unwrap_or(low.vimgrep); + let heading = match low.heading { + None => !low.vimgrep && state.is_terminal_stdout, + Some(false) => false, + Some(true) => !low.vimgrep, + }; + let path_terminator = if low.null { Some(b'\x00') } else { None }; + let quit_after_match = stats.is_none() && low.quiet; + let threads = if cfg!(target_os = "wasi") || low.sort.is_some() || paths.is_one_file { + 1 + } else if let Some(threads) = low.threads { + threads + } else { + std::thread::available_parallelism().map_or(1, |n| n.get()).min(12) + }; + log::debug!("using {threads} thread(s)"); + let with_filename = low + .with_filename + .unwrap_or_else(|| low.vimgrep || !paths.is_one_file); + + let file_separator = match low.mode { + Mode::Search(SearchMode::Standard) => { + if heading { + Some(b"".to_vec()) + } else if let ContextMode::Limited(ref limited) = low.context { + let (before, after) = limited.get(); + if before > 0 || after > 0 { + low.context_separator.clone().into_bytes() + } else { + None + } + } else { + None + } + } + _ => None, + }; + + let line_number = low.line_number.unwrap_or_else(|| { + if low.quiet { + return false; + } + let Mode::Search(ref search_mode) = low.mode else { return false }; + match *search_mode { + SearchMode::FilesWithMatches + | SearchMode::FilesWithoutMatch + | SearchMode::Count + | SearchMode::CountMatches => return false, + SearchMode::JSON => return true, + SearchMode::Standard => { + // A few things can imply counting line numbers. In + // particular, we generally want to show line numbers by + // default when printing to a tty for human consumption, + // except for one interesting case: when we're only + // searching stdin. This makes pipelines work as expected. + (state.is_terminal_stdout && !paths.is_only_stdin()) + || column + || low.vimgrep + } + } + }); + + let mmap_choice = { + // SAFETY: Memory maps are difficult to impossible to encapsulate + // safely in a portable way that doesn't simultaneously negate some + // of the benefits of using memory maps. For ripgrep's use, we never + // mutate a memory map and generally never store the contents of + // memory map in a data structure that depends on immutability. + // Generally speaking, the worst thing that can happen is a SIGBUS + // (if the underlying file is truncated while reading it), which + // will cause ripgrep to abort. This reasoning should be treated as + // suspect. + let maybe = unsafe { grep::searcher::MmapChoice::auto() }; + let never = grep::searcher::MmapChoice::never(); + match low.mmap { + MmapMode::Auto => { + if paths.paths.len() <= 10 + && paths.paths.iter().all(|p| p.is_file()) + { + // If we're only searching a few paths and all of them + // are files, then memory maps are probably faster. + maybe + } else { + never + } + } + MmapMode::AlwaysTryMmap => maybe, + MmapMode::Never => never, + } + }; + + Ok(HiArgs { + mode: low.mode, + patterns, + paths, + binary, + boundary: low.boundary, + buffer: low.buffer, + byte_offset: low.byte_offset, + case: low.case, + color, + colors, + column, + context: low.context, + context_separator: low.context_separator, + crlf: low.crlf, + cwd: state.cwd, + dfa_size_limit: low.dfa_size_limit, + encoding: low.encoding, + engine: low.engine, + field_context_separator: low.field_context_separator, + field_match_separator: low.field_match_separator, + file_separator, + fixed_strings: low.fixed_strings, + follow: low.follow, + heading, + hidden: low.hidden, + hyperlink_config, + ignore_file: low.ignore_file, + ignore_file_case_insensitive: low.ignore_file_case_insensitive, + include_zero: low.include_zero, + invert_match: low.invert_match, + is_terminal_stdout: state.is_terminal_stdout, + line_number, + max_columns: low.max_columns, + max_columns_preview: low.max_columns_preview, + max_count: low.max_count, + max_depth: low.max_depth, + max_filesize: low.max_filesize, + mmap_choice, + multiline: low.multiline, + multiline_dotall: low.multiline_dotall, + no_ignore_dot: low.no_ignore_dot, + no_ignore_exclude: low.no_ignore_exclude, + no_ignore_files: low.no_ignore_files, + no_ignore_global: low.no_ignore_global, + no_ignore_parent: low.no_ignore_parent, + no_ignore_vcs: low.no_ignore_vcs, + no_require_git: low.no_require_git, + no_unicode: low.no_unicode, + null_data: low.null_data, + one_file_system: low.one_file_system, + only_matching: low.only_matching, + globs, + path_separator: low.path_separator, + path_terminator, + pre: low.pre, + pre_globs, + quiet: low.quiet, + quit_after_match, + regex_size_limit: low.regex_size_limit, + replace: low.replace, + search_zip: low.search_zip, + sort: low.sort, + stats, + stop_on_nonmatch: low.stop_on_nonmatch, + threads, + trim: low.trim, + types, + vimgrep: low.vimgrep, + with_filename, + }) + } + + /// Returns a writer for printing buffers to stdout. + /// + /// This is intended to be used from multiple threads. Namely, a buffer + /// writer can create new buffers that are sent to threads. Threads can + /// then independently write to the buffers. Once a unit of work is + /// complete, a buffer can be given to the buffer writer to write to + /// stdout. + pub(crate) fn buffer_writer(&self) -> termcolor::BufferWriter { + let mut wtr = + termcolor::BufferWriter::stdout(self.color.to_termcolor()); + wtr.separator(self.file_separator.clone()); + wtr + } + + /// Returns true when ripgrep had to guess to search the current working + /// directory. That is, it's true when ripgrep is called without any file + /// paths or directories to search. + /// + /// Other than changing how file paths are printed (i.e., without the + /// leading `./`), it's also useful to know for diagnostic reasons. For + /// example, ripgrep will print an error message when nothing is searched + /// since it's possible the ignore rules in play are too aggressive. But + /// this warning is only emitted when ripgrep was called without any + /// explicit file paths since otherwise the warning would likely be too + /// aggressive. + pub(crate) fn has_implicit_path(&self) -> bool { + self.paths.has_implicit_path + } + + /// Return a properly configured builder for constructing haystacks. + /// + /// The builder can be used to turn a directory entry (from the `ignore` + /// crate) into something that can be searched. + pub(crate) fn haystack_builder(&self) -> HaystackBuilder { + let mut builder = HaystackBuilder::new(); + builder.strip_dot_prefix(self.paths.has_implicit_path); + builder + } + + /// Return the matcher that should be used for searching using the engine + /// choice made by the user. + /// + /// If there was a problem building the matcher (e.g., a syntax error), + /// then this returns an error. + pub(crate) fn matcher(&self) -> anyhow::Result { + match self.engine { + EngineChoice::Default => match self.matcher_rust() { + Ok(m) => Ok(m), + Err(err) => { + anyhow::bail!(suggest_other_engine(err.to_string())); + } + }, + EngineChoice::PCRE2 => Ok(self.matcher_pcre2()?), + EngineChoice::Auto => { + let rust_err = match self.matcher_rust() { + Ok(m) => return Ok(m), + Err(err) => err, + }; + log::debug!( + "error building Rust regex in hybrid mode:\n{rust_err}", + ); + + let pcre_err = match self.matcher_pcre2() { + Ok(m) => return Ok(m), + Err(err) => err, + }; + let divider = "~".repeat(79); + anyhow::bail!( + "regex could not be compiled with either the default \ + regex engine or with PCRE2.\n\n\ + default regex engine error:\n\ + {divider}\n\ + {rust_err}\n\ + {divider}\n\n\ + PCRE2 regex engine error:\n{pcre_err}", + ); + } + } + } + + /// Build a matcher using PCRE2. + /// + /// If there was a problem building the matcher (such as a regex syntax + /// error), then an error is returned. + /// + /// If the `pcre2` feature is not enabled then this always returns an + /// error. + fn matcher_pcre2(&self) -> anyhow::Result { + #[cfg(feature = "pcre2")] + { + let mut builder = grep::pcre2::RegexMatcherBuilder::new(); + builder.multi_line(true).fixed_strings(self.fixed_strings); + match self.case { + CaseMode::Sensitive => builder.caseless(false), + CaseMode::Insensitive => builder.caseless(true), + CaseMode::Smart => builder.case_smart(true), + }; + if let Some(ref boundary) = self.boundary { + match *boundary { + BoundaryMode::Line => builder.whole_line(true), + BoundaryMode::Word => builder.word(true), + }; + } + // For whatever reason, the JIT craps out during regex compilation with + // a "no more memory" error on 32 bit systems. So don't use it there. + if cfg!(target_pointer_width = "64") { + builder + .jit_if_available(true) + // The PCRE2 docs say that 32KB is the default, and that 1MB + // should be big enough for anything. But let's crank it to + // 10MB. + .max_jit_stack_size(Some(10 * (1 << 20))); + } + if !self.no_unicode { + builder.utf(true).ucp(true); + } + if self.multiline { + builder.dotall(self.multiline_dotall); + } + if self.crlf { + builder.crlf(true); + } + let m = builder.build_many(&self.patterns.patterns)?; + Ok(PatternMatcher::PCRE2(m)) + } + #[cfg(not(feature = "pcre2"))] + { + Err(anyhow::anyhow!( + "PCRE2 is not available in this build of ripgrep" + )) + } + } + + /// Build a matcher using Rust's regex engine. + /// + /// If there was a problem building the matcher (such as a regex syntax + /// error), then an error is returned. + fn matcher_rust(&self) -> anyhow::Result { + let mut builder = grep::regex::RegexMatcherBuilder::new(); + builder + .multi_line(true) + .unicode(!self.no_unicode) + .octal(false) + .fixed_strings(self.fixed_strings); + match self.case { + CaseMode::Sensitive => builder.case_insensitive(false), + CaseMode::Insensitive => builder.case_insensitive(true), + CaseMode::Smart => builder.case_smart(true), + }; + if let Some(ref boundary) = self.boundary { + match *boundary { + BoundaryMode::Line => builder.whole_line(true), + BoundaryMode::Word => builder.word(true), + }; + } + if self.multiline { + builder.dot_matches_new_line(self.multiline_dotall); + if self.crlf { + builder.crlf(true).line_terminator(None); + } + } else { + builder.line_terminator(Some(b'\n')).dot_matches_new_line(false); + if self.crlf { + builder.crlf(true); + } + // We don't need to set this in multiline mode since multiline + // matchers don't use optimizations related to line terminators. + // Moreover, a multiline regex used with --null-data should + // be allowed to match NUL bytes explicitly, which this would + // otherwise forbid. + if self.null_data { + builder.line_terminator(Some(b'\x00')); + } + } + if let Some(limit) = self.regex_size_limit { + builder.size_limit(limit); + } + if let Some(limit) = self.dfa_size_limit { + builder.dfa_size_limit(limit); + } + if !self.binary.is_none() { + builder.ban_byte(Some(b'\x00')); + } + let m = match builder.build_many(&self.patterns.patterns) { + Ok(m) => m, + Err(err) => { + anyhow::bail!(suggest_text(suggest_multiline(err.to_string()))) + } + }; + Ok(PatternMatcher::RustRegex(m)) + } + + /// Returns true if some non-zero number of matches is believed to be + /// possible. + /// + /// When this returns false, it is impossible for ripgrep to ever report + /// a match. + pub(crate) fn matches_possible(&self) -> bool { + if self.patterns.patterns.is_empty() && !self.invert_match { + return false; + } + if self.max_count == Some(0) { + return false; + } + true + } + + /// Returns the "mode" that ripgrep should operate in. + /// + /// This is generally useful for determining what action ripgrep should + /// take. The main mode is of course to "search," but there are other + /// non-search modes such as `--type-list` and `--files`. + pub(crate) fn mode(&self) -> Mode { + self.mode + } + + /// Returns a builder for constructing a "path printer." + /// + /// This is useful for the `--files` mode in ripgrep, where the printer + /// just needs to emit paths and not need to worry about the functionality + /// of searching. + pub(crate) fn path_printer_builder( + &self, + ) -> grep::printer::PathPrinterBuilder { + let mut builder = grep::printer::PathPrinterBuilder::new(); + builder + .color_specs(self.colors.clone()) + .hyperlink(self.hyperlink_config.clone()) + .separator(self.path_separator.clone()) + .terminator(self.path_terminator.unwrap_or(b'\n')); + builder + } + + /// Returns a printer for the given search mode. + /// + /// This chooses which printer to build (JSON, summary or standard) based + /// on the search mode given. + pub(crate) fn printer( + &self, + search_mode: SearchMode, + wtr: W, + ) -> Printer { + let summary_kind = if self.quiet { + match search_mode { + SearchMode::FilesWithMatches + | SearchMode::Count + | SearchMode::CountMatches + | SearchMode::JSON + | SearchMode::Standard => SummaryKind::QuietWithMatch, + SearchMode::FilesWithoutMatch => { + SummaryKind::QuietWithoutMatch + } + } + } else { + match search_mode { + SearchMode::FilesWithMatches => SummaryKind::PathWithMatch, + SearchMode::FilesWithoutMatch => SummaryKind::PathWithoutMatch, + SearchMode::Count => SummaryKind::Count, + SearchMode::CountMatches => SummaryKind::CountMatches, + SearchMode::JSON => { + return Printer::JSON(self.printer_json(wtr)); + } + SearchMode::Standard => { + return Printer::Standard(self.printer_standard(wtr)); + } + } + }; + Printer::Summary(self.printer_summary(wtr, summary_kind)) + } + + /// Builds a JSON printer. + fn printer_json( + &self, + wtr: W, + ) -> grep::printer::JSON { + grep::printer::JSONBuilder::new() + .pretty(false) + .always_begin_end(false) + .replacement(self.replace.clone().map(|r| r.into())) + .build(wtr) + } + + /// Builds a "standard" grep printer where matches are printed as plain + /// text lines. + fn printer_standard( + &self, + wtr: W, + ) -> grep::printer::Standard { + let mut builder = grep::printer::StandardBuilder::new(); + builder + .byte_offset(self.byte_offset) + .color_specs(self.colors.clone()) + .column(self.column) + .heading(self.heading) + .hyperlink(self.hyperlink_config.clone()) + .max_columns_preview(self.max_columns_preview) + .max_columns(self.max_columns) + .only_matching(self.only_matching) + .path(self.with_filename) + .path_terminator(self.path_terminator.clone()) + .per_match_one_line(true) + .per_match(self.vimgrep) + .replacement(self.replace.clone().map(|r| r.into())) + .separator_context(self.context_separator.clone().into_bytes()) + .separator_field_context( + self.field_context_separator.clone().into_bytes(), + ) + .separator_field_match( + self.field_match_separator.clone().into_bytes(), + ) + .separator_path(self.path_separator.clone()) + .stats(self.stats.is_some()) + .trim_ascii(self.trim); + // When doing multi-threaded searching, the buffer writer is + // responsible for writing separators since it is the only thing that + // knows whether something has been printed or not. But for the single + // threaded case, we don't use a buffer writer and thus can let the + // printer own this. + if self.threads == 1 { + builder.separator_search(self.file_separator.clone()); + } + builder.build(wtr) + } + + /// Builds a "summary" printer where search results are aggregated on a + /// file-by-file basis. + fn printer_summary( + &self, + wtr: W, + kind: SummaryKind, + ) -> grep::printer::Summary { + grep::printer::SummaryBuilder::new() + .color_specs(self.colors.clone()) + .exclude_zero(!self.include_zero) + .hyperlink(self.hyperlink_config.clone()) + .kind(kind) + .path(self.with_filename) + .path_terminator(self.path_terminator.clone()) + .separator_field(b":".to_vec()) + .separator_path(self.path_separator.clone()) + .stats(self.stats.is_some()) + .build(wtr) + } + + /// Returns true if ripgrep should operate in "quiet" mode. + /// + /// Generally speaking, quiet mode means that ripgrep should not print + /// anything to stdout. There are some exceptions. For example, when the + /// user has provided `--stats`, then ripgrep will print statistics to + /// stdout. + pub(crate) fn quiet(&self) -> bool { + self.quiet + } + + /// Returns true when ripgrep should stop searching after a single match is + /// found. + /// + /// This is useful for example when quiet mode is enabled. In that case, + /// users generally can't tell the difference in behavior between a search + /// that finds all matches and a search that only finds one of them. (An + /// exception here is if `--stats` is given, then `quit_after_match` will + /// always return false since the user expects ripgrep to find everything.) + pub(crate) fn quit_after_match(&self) -> bool { + self.quit_after_match + } + + /// Build a worker for executing searches. + /// + /// Search results are found using the given matcher and written to the + /// given printer. + pub(crate) fn search_worker( + &self, + matcher: PatternMatcher, + searcher: grep::searcher::Searcher, + printer: Printer, + ) -> anyhow::Result> { + let mut builder = SearchWorkerBuilder::new(); + builder + .preprocessor(self.pre.clone())? + .preprocessor_globs(self.pre_globs.clone()) + .search_zip(self.search_zip) + .binary_detection_explicit(self.binary.explicit.clone()) + .binary_detection_implicit(self.binary.implicit.clone()); + Ok(builder.build(matcher, searcher, printer)) + } + + /// Build a searcher from the command line parameters. + pub(crate) fn searcher(&self) -> anyhow::Result { + let line_term = if self.crlf { + grep::matcher::LineTerminator::crlf() + } else if self.null_data { + grep::matcher::LineTerminator::byte(b'\x00') + } else { + grep::matcher::LineTerminator::byte(b'\n') + }; + let mut builder = grep::searcher::SearcherBuilder::new(); + builder + .max_matches(self.max_count) + .line_terminator(line_term) + .invert_match(self.invert_match) + .line_number(self.line_number) + .multi_line(self.multiline) + .memory_map(self.mmap_choice.clone()) + .stop_on_nonmatch(self.stop_on_nonmatch); + match self.context { + ContextMode::Passthru => { + builder.passthru(true); + } + ContextMode::Limited(ref limited) => { + let (before, after) = limited.get(); + builder.before_context(before); + builder.after_context(after); + } + } + match self.encoding { + EncodingMode::Auto => {} // default for the searcher + EncodingMode::Some(ref enc) => { + builder.encoding(Some(enc.clone())); + } + EncodingMode::Disabled => { + builder.bom_sniffing(false); + } + } + Ok(builder.build()) + } + + /// Given an iterator of haystacks, sort them if necessary. + /// + /// When sorting is necessary, this will collect the entire iterator into + /// memory, sort them and then return a new iterator. When sorting is not + /// necessary, then the iterator given is returned as is without collecting + /// it into memory. + /// + /// Once special case is when sorting by path in ascending order has been + /// requested. In this case, the iterator given is returned as is without + /// any additional sorting. This is done because `walk_builder()` will sort + /// the iterator it yields during directory traversal, so no additional + /// sorting is needed. + pub(crate) fn sort<'a, I>( + &self, + haystacks: I, + ) -> Box + 'a> + where + I: Iterator + 'a, + { + use std::{cmp::Ordering, fs::Metadata, io, time::SystemTime}; + + fn attach_timestamps( + haystacks: impl Iterator, + get: impl Fn(&Metadata) -> io::Result, + ) -> impl Iterator)> { + haystacks.map(move |s| { + let time = s.path().metadata().and_then(|m| get(&m)).ok(); + (s, time) + }) + } + + let Some(ref sort) = self.sort else { return Box::new(haystacks) }; + let mut with_timestamps: Vec<_> = match sort.kind { + SortModeKind::Path if !sort.reverse => return Box::new(haystacks), + SortModeKind::Path => { + let mut haystacks = haystacks.collect::>(); + haystacks.sort_by(|ref h1, ref h2| { + h1.path().cmp(h2.path()).reverse() + }); + return Box::new(haystacks.into_iter()); + } + SortModeKind::LastModified => { + attach_timestamps(haystacks, |md| md.modified()).collect() + } + SortModeKind::LastAccessed => { + attach_timestamps(haystacks, |md| md.accessed()).collect() + } + SortModeKind::Created => { + attach_timestamps(haystacks, |md| md.created()).collect() + } + }; + with_timestamps.sort_by(|(_, t1), (_, t2)| { + let ordering = match (*t1, *t2) { + // Both have metadata, do the obvious thing. + (Some(t1), Some(t2)) => t1.cmp(&t2), + // Things that error should appear later (when ascending). + (Some(_), None) => Ordering::Less, + // Things that error should appear later (when ascending). + (None, Some(_)) => Ordering::Greater, + // When both error, we can't distinguish, so treat as equal. + (None, None) => Ordering::Equal, + }; + if sort.reverse { ordering.reverse() } else { ordering } + }); + Box::new(with_timestamps.into_iter().map(|(s, _)| s)) + } + + /// Returns a stats object if the user requested that ripgrep keep track + /// of various metrics during a search. + /// + /// When this returns `None`, then callers may assume that the user did + /// not request statistics. + pub(crate) fn stats(&self) -> Option { + self.stats.clone() + } + + /// Returns a color-enabled writer for stdout. + /// + /// The writer returned is also configured to do either line or block + /// buffering, based on either explicit configuration from the user via CLI + /// flags, or automatically based on whether stdout is connected to a tty. + pub(crate) fn stdout(&self) -> grep::cli::StandardStream { + let color = self.color.to_termcolor(); + match self.buffer { + BufferMode::Auto => { + if self.is_terminal_stdout { + grep::cli::stdout_buffered_line(color) + } else { + grep::cli::stdout_buffered_block(color) + } + } + BufferMode::Line => grep::cli::stdout_buffered_line(color), + BufferMode::Block => grep::cli::stdout_buffered_block(color), + } + } + + /// Returns the total number of threads ripgrep should use to execute a + /// search. + /// + /// This number is the result of reasoning about both heuristics (like + /// the available number of cores) and whether ripgrep's mode supports + /// parallelism. It is intended that this number be used to directly + /// determine how many threads to spawn. + pub(crate) fn threads(&self) -> usize { + self.threads + } + + /// Returns the file type matcher that was built. + /// + /// The matcher includes both the default rules and any rules added by the + /// user for this specific invocation. + pub(crate) fn types(&self) -> &ignore::types::Types { + &self.types + } + + /// Create a new builder for recursive directory traversal. + /// + /// The builder returned can be used to start a single threaded or multi + /// threaded directory traversal. For multi threaded traversal, the number + /// of threads configured is equivalent to `HiArgs::threads`. + /// + /// If `HiArgs::threads` is equal to `1`, then callers should generally + /// choose to explicitly use single threaded traversal since it won't have + /// the unnecessary overhead of synchronization. + pub(crate) fn walk_builder(&self) -> anyhow::Result { + let mut builder = ignore::WalkBuilder::new(&self.paths.paths[0]); + for path in self.paths.paths.iter().skip(1) { + builder.add(path); + } + if !self.no_ignore_files { + for path in self.ignore_file.iter() { + if let Some(err) = builder.add_ignore(path) { + ignore_message!("{err}"); + } + } + } + builder + .max_depth(self.max_depth) + .follow_links(self.follow) + .max_filesize(self.max_filesize) + .threads(self.threads) + .same_file_system(self.one_file_system) + .skip_stdout(matches!(self.mode, Mode::Search(_))) + .overrides(self.globs.clone()) + .types(self.types.clone()) + .hidden(!self.hidden) + .parents(!self.no_ignore_parent) + .ignore(!self.no_ignore_dot) + .git_global(!self.no_ignore_vcs && !self.no_ignore_global) + .git_ignore(!self.no_ignore_vcs) + .git_exclude(!self.no_ignore_vcs && !self.no_ignore_exclude) + .require_git(!self.no_require_git) + .ignore_case_insensitive(self.ignore_file_case_insensitive) + .current_dir(&self.cwd); + if !self.no_ignore_dot { + builder.add_custom_ignore_filename(".rgignore"); + } + // When we want to sort paths lexicographically in ascending order, + // then we can actually do this during directory traversal itself. + // Otherwise, sorting is done by collecting all paths, sorting them and + // then searching them. + if let Some(ref sort) = self.sort { + assert_eq!(1, self.threads, "sorting implies single threaded"); + if !sort.reverse && matches!(sort.kind, SortModeKind::Path) { + builder.sort_by_file_name(|a, b| a.cmp(b)); + } + } + Ok(builder) + } +} + +/// State that only needs to be computed once during argument parsing. +/// +/// This state is meant to be somewhat generic and shared across multiple +/// low->high argument conversions. The state can even be mutated by various +/// conversions as a way to communicate changes to other conversions. For +/// example, reading patterns might consume from stdin. If we know stdin +/// has been consumed and no other file paths have been given, then we know +/// for sure that we should search the CWD. In this way, a state change +/// when reading the patterns can impact how the file paths are ultimately +/// generated. +#[derive(Debug)] +struct State { + /// Whether it's believed that tty is connected to stdout. Note that on + /// unix systems, this is always correct. On Windows, heuristics are used + /// by Rust's standard library, particularly for cygwin/MSYS environments. + is_terminal_stdout: bool, + /// Whether stdin has already been consumed. This is useful to know and for + /// providing good error messages when the user has tried to read from stdin + /// in two different places. For example, `rg -f - -`. + stdin_consumed: bool, + /// The current working directory. + cwd: PathBuf, +} + +impl State { + /// Initialize state to some sensible defaults. + /// + /// Note that the state values may change throughout the lifetime of + /// argument parsing. + fn new() -> anyhow::Result { + use std::io::IsTerminal; + + let cwd = current_dir()?; + log::debug!("read CWD from environment: {}", cwd.display()); + Ok(State { + is_terminal_stdout: std::io::stdout().is_terminal(), + stdin_consumed: false, + cwd, + }) + } +} + +/// The disjunction of patterns to search for. +/// +/// The number of patterns can be empty, e.g., via `-f /dev/null`. +#[derive(Debug)] +struct Patterns { + /// The actual patterns to match. + patterns: Vec, +} + +impl Patterns { + /// Pulls the patterns out of the low arguments. + /// + /// This includes collecting patterns from -e/--regexp and -f/--file. + /// + /// If the invocation implies that the first positional argument is a + /// pattern (the common case), then the first positional argument is + /// extracted as well. + fn from_low_args( + state: &mut State, + low: &mut LowArgs, + ) -> anyhow::Result { + // The first positional is only a pattern when ripgrep is instructed to + // search and neither -e/--regexp nor -f/--file is given. Basically, + // the first positional is a pattern only when a pattern hasn't been + // given in some other way. + + // No search means no patterns. Even if -e/--regexp or -f/--file is + // given, we know we won't use them so don't bother collecting them. + if !matches!(low.mode, Mode::Search(_)) { + return Ok(Patterns { patterns: vec![] }); + } + // If we got nothing from -e/--regexp and -f/--file, then the first + // positional is a pattern. + if low.patterns.is_empty() { + anyhow::ensure!( + !low.positional.is_empty(), + "ripgrep requires at least one pattern to execute a search" + ); + let ospat = low.positional.remove(0); + let Ok(pat) = ospat.into_string() else { + anyhow::bail!("pattern given is not valid UTF-8") + }; + return Ok(Patterns { patterns: vec![pat] }); + } + // Otherwise, we need to slurp up our patterns from -e/--regexp and + // -f/--file. We de-duplicate as we go. If we don't de-duplicate, + // then it can actually lead to major slow downs for sloppy inputs. + // This might be surprising, and the regex engine will eventually + // de-duplicate duplicative branches in a single regex (maybe), but + // not until after it has gone through parsing and some other layers. + // If there are a lot of duplicates, then that can lead to a sizeable + // extra cost. It is lamentable that we pay the extra cost here to + // de-duplicate for a likely uncommon case, but I've seen this have a + // big impact on real world data. + let mut seen = HashSet::new(); + let mut patterns = Vec::with_capacity(low.patterns.len()); + let mut add = |pat: String| { + if !seen.contains(&pat) { + seen.insert(pat.clone()); + patterns.push(pat); + } + }; + for source in low.patterns.drain(..) { + match source { + PatternSource::Regexp(pat) => add(pat), + PatternSource::File(path) => { + if path == Path::new("-") { + anyhow::ensure!( + !state.stdin_consumed, + "error reading -f/--file from stdin: stdin \ + has already been consumed" + ); + for pat in grep::cli::patterns_from_stdin()? { + add(pat); + } + state.stdin_consumed = true; + } else { + for pat in grep::cli::patterns_from_path(&path)? { + add(pat); + } + } + } + } + } + Ok(Patterns { patterns }) + } +} + +/// The collection of paths we want to search for. +/// +/// This guarantees that there is always at least one path. +#[derive(Debug)] +struct Paths { + /// The actual paths. + paths: Vec, + /// This is true when ripgrep had to guess to search the current working + /// directory. e.g., When the user just runs `rg foo`. It is odd to need + /// this, but it subtly changes how the paths are printed. When no explicit + /// path is given, then ripgrep doesn't prefix each path with `./`. But + /// otherwise it does! This curious behavior matches what GNU grep does. + has_implicit_path: bool, + /// Set to true if it is known that only a single file descriptor will + /// be searched. + is_one_file: bool, +} + +impl Paths { + /// Drain the search paths out of the given low arguments. + fn from_low_args( + state: &mut State, + _: &Patterns, + low: &mut LowArgs, + ) -> anyhow::Result { + // We require a `&Patterns` even though we don't use it to ensure that + // patterns have already been read from LowArgs. This let's us safely + // assume that all remaining positional arguments are intended to be + // file paths. + + let mut paths = Vec::with_capacity(low.positional.len()); + for osarg in low.positional.drain(..) { + let path = PathBuf::from(osarg); + if state.stdin_consumed && path == Path::new("-") { + anyhow::bail!( + "error: attempted to read patterns from stdin \ + while also searching stdin", + ); + } + paths.push(path); + } + log::debug!("number of paths given to search: {}", paths.len()); + if !paths.is_empty() { + let is_one_file = paths.len() == 1 + // Note that we specifically use `!paths[0].is_dir()` here + // instead of `paths[0].is_file()`. Namely, the latter can + // return `false` even when the path is something resembling + // a file. So instead, we just consider the path a file as + // long as we know it isn't a directory. + // + // See: https://github.com/BurntSushi/ripgrep/issues/2736 + && (paths[0] == Path::new("-") || !paths[0].is_dir()); + log::debug!("is_one_file? {is_one_file:?}"); + return Ok(Paths { paths, has_implicit_path: false, is_one_file }); + } + // N.B. is_readable_stdin is a heuristic! Part of the issue is that a + // lot of "exec process" APIs will open a stdin pipe even though stdin + // isn't really being used. ripgrep then thinks it should search stdin + // and one gets the appearance of it hanging. It's a terrible failure + // mode, but there really is no good way to mitigate it. It's just a + // consequence of letting the user type 'rg foo' and "guessing" that + // they meant to search the CWD. + let is_readable_stdin = grep::cli::is_readable_stdin(); + let use_cwd = !is_readable_stdin + || state.stdin_consumed + || !matches!(low.mode, Mode::Search(_)); + log::debug!( + "using heuristics to determine whether to read from \ + stdin or search ./ (\ + is_readable_stdin={is_readable_stdin}, \ + stdin_consumed={stdin_consumed}, \ + mode={mode:?})", + stdin_consumed = state.stdin_consumed, + mode = low.mode, + ); + let (path, is_one_file) = if use_cwd { + log::debug!("heuristic chose to search ./"); + (PathBuf::from("./"), false) + } else { + log::debug!("heuristic chose to search stdin"); + (PathBuf::from("-"), true) + }; + Ok(Paths { paths: vec![path], has_implicit_path: true, is_one_file }) + } + + /// Returns true if ripgrep will only search stdin and nothing else. + fn is_only_stdin(&self) -> bool { + self.paths.len() == 1 && self.paths[0] == Path::new("-") + } +} + +/// The "binary detection" configuration that ripgrep should use. +/// +/// ripgrep actually uses two different binary detection heuristics depending +/// on whether a file is explicitly being searched (e.g., via a CLI argument) +/// or implicitly searched (e.g., via directory traversal). In general, the +/// former can never use a heuristic that lets it "quit" searching before +/// either getting EOF or finding a match. (Because doing otherwise would be +/// considered a filter, and ripgrep follows the rule that an explicitly given +/// file is always searched.) +#[derive(Debug)] +struct BinaryDetection { + explicit: grep::searcher::BinaryDetection, + implicit: grep::searcher::BinaryDetection, +} + +impl BinaryDetection { + /// Determines the correct binary detection mode from low-level arguments. + fn from_low_args(_: &State, low: &LowArgs) -> BinaryDetection { + let none = matches!(low.binary, BinaryMode::AsText) || low.null_data; + let convert = matches!(low.binary, BinaryMode::SearchAndSuppress); + let explicit = if none { + grep::searcher::BinaryDetection::none() + } else { + grep::searcher::BinaryDetection::convert(b'\x00') + }; + let implicit = if none { + grep::searcher::BinaryDetection::none() + } else if convert { + grep::searcher::BinaryDetection::convert(b'\x00') + } else { + grep::searcher::BinaryDetection::quit(b'\x00') + }; + BinaryDetection { explicit, implicit } + } + + /// Returns true when both implicit and explicit binary detection is + /// disabled. + pub(crate) fn is_none(&self) -> bool { + let none = grep::searcher::BinaryDetection::none(); + self.explicit == none && self.implicit == none + } +} + +/// Builds the file type matcher from low level arguments. +fn types(low: &LowArgs) -> anyhow::Result { + let mut builder = ignore::types::TypesBuilder::new(); + builder.add_defaults(); + for tychange in low.type_changes.iter() { + match *tychange { + TypeChange::Clear { ref name } => { + builder.clear(name); + } + TypeChange::Add { ref def } => { + builder.add_def(def)?; + } + TypeChange::Select { ref name } => { + builder.select(name); + } + TypeChange::Negate { ref name } => { + builder.negate(name); + } + } + } + Ok(builder.build()?) +} + +/// Builds the glob "override" matcher from the CLI `-g/--glob` and `--iglob` +/// flags. +fn globs( + state: &State, + low: &LowArgs, +) -> anyhow::Result { + if low.globs.is_empty() && low.iglobs.is_empty() { + return Ok(ignore::overrides::Override::empty()); + } + let mut builder = ignore::overrides::OverrideBuilder::new(&state.cwd); + // Make all globs case insensitive with --glob-case-insensitive. + if low.glob_case_insensitive { + builder.case_insensitive(true).unwrap(); + } + for glob in low.globs.iter() { + builder.add(glob)?; + } + // This only enables case insensitivity for subsequent globs. + builder.case_insensitive(true).unwrap(); + for glob in low.iglobs.iter() { + builder.add(&glob)?; + } + Ok(builder.build()?) +} + +/// Builds a glob matcher for all of the preprocessor globs (via `--pre-glob`). +fn preprocessor_globs( + state: &State, + low: &LowArgs, +) -> anyhow::Result { + if low.pre_glob.is_empty() { + return Ok(ignore::overrides::Override::empty()); + } + let mut builder = ignore::overrides::OverrideBuilder::new(&state.cwd); + for glob in low.pre_glob.iter() { + builder.add(glob)?; + } + Ok(builder.build()?) +} + +/// Determines whether stats should be tracked for this search. If so, a stats +/// object is returned. +fn stats(low: &LowArgs) -> Option { + if !matches!(low.mode, Mode::Search(_)) { + return None; + } + if low.stats || matches!(low.mode, Mode::Search(SearchMode::JSON)) { + return Some(grep::printer::Stats::new()); + } + None +} + +/// Pulls out any color specs provided by the user and assembles them into one +/// single configuration. +fn take_color_specs(_: &mut State, low: &mut LowArgs) -> ColorSpecs { + let mut specs = grep::printer::default_color_specs(); + for spec in low.colors.drain(..) { + specs.push(spec); + } + ColorSpecs::new(&specs) +} + +/// Pulls out the necessary info from the low arguments to build a full +/// hyperlink configuration. +fn take_hyperlink_config( + _: &mut State, + low: &mut LowArgs, +) -> anyhow::Result { + let mut env = grep::printer::HyperlinkEnvironment::new(); + if let Some(hostname) = hostname(low.hostname_bin.as_deref()) { + log::debug!("found hostname for hyperlink configuration: {hostname}"); + env.host(Some(hostname)); + } + if let Some(wsl_prefix) = wsl_prefix() { + log::debug!( + "found wsl_prefix for hyperlink configuration: {wsl_prefix}" + ); + env.wsl_prefix(Some(wsl_prefix)); + } + let fmt = std::mem::take(&mut low.hyperlink_format); + log::debug!("hyperlink format: {:?}", fmt.to_string()); + Ok(grep::printer::HyperlinkConfig::new(env, fmt)) +} + +/// Attempts to discover the current working directory. +/// +/// This mostly just defers to the standard library, however, such things will +/// fail if ripgrep is in a directory that no longer exists. We attempt some +/// fallback mechanisms, such as querying the PWD environment variable, but +/// otherwise return an error. +fn current_dir() -> anyhow::Result { + let err = match std::env::current_dir() { + Err(err) => err, + Ok(cwd) => return Ok(cwd), + }; + if let Some(cwd) = std::env::var_os("PWD") { + if !cwd.is_empty() { + return Ok(PathBuf::from(cwd)); + } + } + anyhow::bail!( + "failed to get current working directory: {err}\n\ + did your CWD get deleted?", + ) +} + +/// Retrieves the hostname that should be used wherever a hostname is required. +/// +/// Currently, this is only used in the hyperlink format. +/// +/// This works by first running the given binary program (if present and with +/// no arguments) to get the hostname after trimming leading and trailing +/// whitespace. If that fails for any reason, then it falls back to getting +/// the hostname via platform specific means (e.g., `gethostname` on Unix). +/// +/// The purpose of `bin` is to make it possible for end users to override how +/// ripgrep determines the hostname. +fn hostname(bin: Option<&Path>) -> Option { + let Some(bin) = bin else { return platform_hostname() }; + let bin = match grep::cli::resolve_binary(bin) { + Ok(bin) => bin, + Err(err) => { + log::debug!( + "failed to run command '{bin:?}' to get hostname \ + (falling back to platform hostname): {err}", + ); + return platform_hostname(); + } + }; + let mut cmd = std::process::Command::new(&bin); + cmd.stdin(std::process::Stdio::null()); + let rdr = match grep::cli::CommandReader::new(&mut cmd) { + Ok(rdr) => rdr, + Err(err) => { + log::debug!( + "failed to spawn command '{bin:?}' to get \ + hostname (falling back to platform hostname): {err}", + ); + return platform_hostname(); + } + }; + let out = match std::io::read_to_string(rdr) { + Ok(out) => out, + Err(err) => { + log::debug!( + "failed to read output from command '{bin:?}' to get \ + hostname (falling back to platform hostname): {err}", + ); + return platform_hostname(); + } + }; + let hostname = out.trim(); + if hostname.is_empty() { + log::debug!( + "output from command '{bin:?}' is empty after trimming \ + leading and trailing whitespace (falling back to \ + platform hostname)", + ); + return platform_hostname(); + } + Some(hostname.to_string()) +} + +/// Attempts to get the hostname by using platform specific routines. +/// +/// For example, this will do `gethostname` on Unix and `GetComputerNameExW` on +/// Windows. +fn platform_hostname() -> Option { + let hostname_os = match grep::cli::hostname() { + Ok(x) => x, + Err(err) => { + log::debug!("could not get hostname: {}", err); + return None; + } + }; + let Some(hostname) = hostname_os.to_str() else { + log::debug!( + "got hostname {:?}, but it's not valid UTF-8", + hostname_os + ); + return None; + }; + Some(hostname.to_string()) +} + +/// Returns the value for the `{wslprefix}` variable in a hyperlink format. +/// +/// A WSL prefix is a share/network like thing that is meant to permit Windows +/// applications to open files stored within a WSL drive. +/// +/// If a WSL distro name is unavailable, not valid UTF-8 or this isn't running +/// in a Unix environment, then this returns None. +/// +/// See: +fn wsl_prefix() -> Option { + if !cfg!(unix) { + return None; + } + let distro_os = std::env::var_os("WSL_DISTRO_NAME")?; + let Some(distro) = distro_os.to_str() else { + log::debug!( + "found WSL_DISTRO_NAME={:?}, but value is not UTF-8", + distro_os + ); + return None; + }; + Some(format!("wsl$/{distro}")) +} + +/// Possibly suggest another regex engine based on the error message given. +/// +/// This inspects an error resulting from building a Rust regex matcher, and +/// if it's believed to correspond to a syntax error that another engine could +/// handle, then add a message to suggest the use of the engine flag. +fn suggest_other_engine(msg: String) -> String { + if let Some(pcre_msg) = suggest_pcre2(&msg) { + return pcre_msg; + } + msg +} + +/// Possibly suggest PCRE2 based on the error message given. +/// +/// Inspect an error resulting from building a Rust regex matcher, and if it's +/// believed to correspond to a syntax error that PCRE2 could handle, then +/// add a message to suggest the use of -P/--pcre2. +fn suggest_pcre2(msg: &str) -> Option { + if !cfg!(feature = "pcre2") { + return None; + } + if !msg.contains("backreferences") && !msg.contains("look-around") { + None + } else { + Some(format!( + "{msg} + +Consider enabling PCRE2 with the --pcre2 flag, which can handle backreferences +and look-around.", + )) + } +} + +/// Possibly suggest multiline mode based on the error message given. +/// +/// Does a bit of a hacky inspection of the given error message, and if it +/// looks like the user tried to type a literal line terminator then it will +/// return a new error message suggesting the use of -U/--multiline. +fn suggest_multiline(msg: String) -> String { + if msg.contains("the literal") && msg.contains("not allowed") { + format!( + "{msg} + +Consider enabling multiline mode with the --multiline flag (or -U for short). +When multiline mode is enabled, new line characters can be matched.", + ) + } else { + msg + } +} + +/// Possibly suggest the `-a/--text` flag. +fn suggest_text(msg: String) -> String { + if msg.contains("pattern contains \"\\0\"") { + format!( + "{msg} + +Consider enabling text mode with the --text flag (or -a for short). Otherwise, +binary detection is enabled and matching a NUL byte is impossible.", + ) + } else { + msg + } +} diff --git a/userland/ripgrep/crates/core/flags/lowargs.rs b/userland/ripgrep/crates/core/flags/lowargs.rs new file mode 100644 index 0000000..a4d4b88 --- /dev/null +++ b/userland/ripgrep/crates/core/flags/lowargs.rs @@ -0,0 +1,723 @@ +/*! +Provides the definition of low level arguments from CLI flags. +*/ + +use std::{ + ffi::{OsStr, OsString}, + path::PathBuf, +}; + +use { + bstr::{BString, ByteVec}, + grep::printer::{HyperlinkFormat, UserColorSpec}, +}; + +/// A collection of "low level" arguments. +/// +/// The "low level" here is meant to constrain this type to be as close to the +/// actual CLI flags and arguments as possible. Namely, other than some +/// convenience types to help validate flag values and deal with overrides +/// between flags, these low level arguments do not contain any higher level +/// abstractions. +/// +/// Another self-imposed constraint is that populating low level arguments +/// should not require anything other than validating what the user has +/// provided. For example, low level arguments should not contain a +/// `HyperlinkConfig`, since in order to get a full configuration, one needs to +/// discover the hostname of the current system (which might require running a +/// binary or a syscall). +/// +/// Low level arguments are populated by the parser directly via the `update` +/// method on the corresponding implementation of the `Flag` trait. +#[derive(Debug, Default)] +pub(crate) struct LowArgs { + // Essential arguments. + pub(crate) special: Option, + pub(crate) mode: Mode, + pub(crate) positional: Vec, + pub(crate) patterns: Vec, + // Everything else, sorted lexicographically. + pub(crate) binary: BinaryMode, + pub(crate) boundary: Option, + pub(crate) buffer: BufferMode, + pub(crate) byte_offset: bool, + pub(crate) case: CaseMode, + pub(crate) color: ColorChoice, + pub(crate) colors: Vec, + pub(crate) column: Option, + pub(crate) context: ContextMode, + pub(crate) context_separator: ContextSeparator, + pub(crate) crlf: bool, + pub(crate) dfa_size_limit: Option, + pub(crate) encoding: EncodingMode, + pub(crate) engine: EngineChoice, + pub(crate) field_context_separator: FieldContextSeparator, + pub(crate) field_match_separator: FieldMatchSeparator, + pub(crate) fixed_strings: bool, + pub(crate) follow: bool, + pub(crate) glob_case_insensitive: bool, + pub(crate) globs: Vec, + pub(crate) heading: Option, + pub(crate) hidden: bool, + pub(crate) hostname_bin: Option, + pub(crate) hyperlink_format: HyperlinkFormat, + pub(crate) iglobs: Vec, + pub(crate) ignore_file: Vec, + pub(crate) ignore_file_case_insensitive: bool, + pub(crate) include_zero: bool, + pub(crate) invert_match: bool, + pub(crate) line_number: Option, + pub(crate) logging: Option, + pub(crate) max_columns: Option, + pub(crate) max_columns_preview: bool, + pub(crate) max_count: Option, + pub(crate) max_depth: Option, + pub(crate) max_filesize: Option, + pub(crate) mmap: MmapMode, + pub(crate) multiline: bool, + pub(crate) multiline_dotall: bool, + pub(crate) no_config: bool, + pub(crate) no_ignore_dot: bool, + pub(crate) no_ignore_exclude: bool, + pub(crate) no_ignore_files: bool, + pub(crate) no_ignore_global: bool, + pub(crate) no_ignore_messages: bool, + pub(crate) no_ignore_parent: bool, + pub(crate) no_ignore_vcs: bool, + pub(crate) no_messages: bool, + pub(crate) no_require_git: bool, + pub(crate) no_unicode: bool, + pub(crate) null: bool, + pub(crate) null_data: bool, + pub(crate) one_file_system: bool, + pub(crate) only_matching: bool, + pub(crate) path_separator: Option, + pub(crate) pre: Option, + pub(crate) pre_glob: Vec, + pub(crate) quiet: bool, + pub(crate) regex_size_limit: Option, + pub(crate) replace: Option, + pub(crate) search_zip: bool, + pub(crate) sort: Option, + pub(crate) stats: bool, + pub(crate) stop_on_nonmatch: bool, + pub(crate) threads: Option, + pub(crate) trim: bool, + pub(crate) type_changes: Vec, + pub(crate) unrestricted: usize, + pub(crate) vimgrep: bool, + pub(crate) with_filename: Option, +} + +/// A "special" mode that supersedes everything else. +/// +/// When one of these modes is present, it overrides everything else and causes +/// ripgrep to short-circuit. In particular, we avoid converting low-level +/// argument types into higher level arguments types that can fail for various +/// reasons related to the environment. (Parsing the low-level arguments can +/// fail too, but usually not in a way that can't be worked around by removing +/// the corresponding arguments from the CLI command.) This is overall a hedge +/// to ensure that version and help information are basically always available. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum SpecialMode { + /// Show a condensed version of "help" output. Generally speaking, this + /// shows each flag and an extremely terse description of that flag on + /// a single line. This corresponds to the `-h` flag. + HelpShort, + /// Shows a very verbose version of the "help" output. The docs for some + /// flags will be paragraphs long. This corresponds to the `--help` flag. + HelpLong, + /// Show condensed version information. e.g., `ripgrep x.y.z`. + VersionShort, + /// Show verbose version information. Includes "short" information as well + /// as features included in the build. + VersionLong, + /// Show PCRE2's version information, or an error if this version of + /// ripgrep wasn't compiled with PCRE2 support. + VersionPCRE2, +} + +/// The overall mode that ripgrep should operate in. +/// +/// If ripgrep were designed without the legacy of grep, these would probably +/// be sub-commands? Perhaps not, since they aren't as frequently used. +/// +/// The point of putting these in one enum is that they are all mutually +/// exclusive and override one another. +/// +/// Note that -h/--help and -V/--version are not included in this because +/// they always overrides everything else, regardless of where it appears +/// in the command line. They are treated as "special" modes that short-circuit +/// ripgrep's usual flow. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum Mode { + /// ripgrep will execute a search of some kind. + Search(SearchMode), + /// Show the files that *would* be searched, but don't actually search + /// them. + Files, + /// List all file type definitions configured, including the default file + /// types and any additional file types added to the command line. + Types, + /// Generate various things like the man page and completion files. + Generate(GenerateMode), +} + +impl Default for Mode { + fn default() -> Mode { + Mode::Search(SearchMode::Standard) + } +} + +impl Mode { + /// Update this mode to the new mode while implementing various override + /// semantics. For example, a search mode cannot override a non-search + /// mode. + pub(crate) fn update(&mut self, new: Mode) { + match *self { + // If we're in a search mode, then anything can override it. + Mode::Search(_) => *self = new, + _ => { + // Once we're in a non-search mode, other non-search modes + // can override it. But search modes cannot. So for example, + // `--files -l` will still be Mode::Files. + if !matches!(*self, Mode::Search(_)) { + *self = new; + } + } + } + } +} + +/// The kind of search that ripgrep is going to perform. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum SearchMode { + /// The default standard mode of operation. ripgrep looks for matches and + /// prints them when found. + /// + /// There is no specific flag for this mode since it's the default. But + /// some of the modes below, like JSON, have negation flags like --no-json + /// that let you revert back to this default mode. + Standard, + /// Show files containing at least one match. + FilesWithMatches, + /// Show files that don't contain any matches. + FilesWithoutMatch, + /// Show files containing at least one match and the number of matching + /// lines. + Count, + /// Show files containing at least one match and the total number of + /// matches. + CountMatches, + /// Print matches in a JSON lines format. + JSON, +} + +/// The thing to generate via the --generate flag. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum GenerateMode { + /// Generate the raw roff used for the man page. + Man, + /// Completions for bash. + CompleteBash, + /// Completions for zsh. + CompleteZsh, + /// Completions for fish. + CompleteFish, + /// Completions for PowerShell. + CompletePowerShell, +} + +/// Indicates how ripgrep should treat binary data. +#[derive(Debug, Default, Eq, PartialEq)] +pub(crate) enum BinaryMode { + /// Automatically determine the binary mode to use. Essentially, when + /// a file is searched explicitly, then it will be searched using the + /// `SearchAndSuppress` strategy. Otherwise, it will be searched in a way + /// that attempts to skip binary files as much as possible. That is, once + /// a file is classified as binary, searching will immediately stop. + #[default] + Auto, + /// Search files even when they have binary data, but if a match is found, + /// suppress it and emit a warning. + /// + /// In this mode, `NUL` bytes are replaced with line terminators. This is + /// a heuristic meant to reduce heap memory usage, since true binary data + /// isn't line oriented. If one attempts to treat such data as line + /// oriented, then one may wind up with impractically large lines. For + /// example, many binary files contain very long runs of NUL bytes. + SearchAndSuppress, + /// Treat all files as if they were plain text. There's no skipping and no + /// replacement of `NUL` bytes with line terminators. + AsText, +} + +/// Indicates what kind of boundary mode to use (line or word). +#[derive(Debug, Eq, PartialEq)] +pub(crate) enum BoundaryMode { + /// Only allow matches when surrounded by line boundaries. + Line, + /// Only allow matches when surrounded by word boundaries. + Word, +} + +/// Indicates the buffer mode that ripgrep should use when printing output. +/// +/// The default is `Auto`. +#[derive(Debug, Default, Eq, PartialEq)] +pub(crate) enum BufferMode { + /// Select the buffer mode, 'line' or 'block', automatically based on + /// whether stdout is connected to a tty. + #[default] + Auto, + /// Flush the output buffer whenever a line terminator is seen. + /// + /// This is useful when wants to see search results more immediately, + /// for example, with `tail -f`. + Line, + /// Flush the output buffer whenever it reaches some fixed size. The size + /// is usually big enough to hold many lines. + /// + /// This is useful for maximum performance, particularly when printing + /// lots of results. + Block, +} + +/// Indicates the case mode for how to interpret all patterns given to ripgrep. +/// +/// The default is `Sensitive`. +#[derive(Debug, Default, Eq, PartialEq)] +pub(crate) enum CaseMode { + /// Patterns are matched case sensitively. i.e., `a` does not match `A`. + #[default] + Sensitive, + /// Patterns are matched case insensitively. i.e., `a` does match `A`. + Insensitive, + /// Patterns are automatically matched case insensitively only when they + /// consist of all lowercase literal characters. For example, the pattern + /// `a` will match `A` but `A` will not match `a`. + Smart, +} + +/// Indicates whether ripgrep should include color/hyperlinks in its output. +/// +/// The default is `Auto`. +#[derive(Debug, Default, Eq, PartialEq)] +pub(crate) enum ColorChoice { + /// Color and hyperlinks will never be used. + Never, + /// Color and hyperlinks will be used only when stdout is connected to a + /// tty. + #[default] + Auto, + /// Color will always be used. + Always, + /// Color will always be used and only ANSI escapes will be used. + /// + /// This only makes sense in the context of legacy Windows console APIs. + /// At time of writing, ripgrep will try to use the legacy console APIs + /// if ANSI coloring isn't believed to be possible. This option will force + /// ripgrep to use ANSI coloring. + Ansi, +} + +impl ColorChoice { + /// Convert this color choice to the corresponding termcolor type. + pub(crate) fn to_termcolor(&self) -> termcolor::ColorChoice { + match *self { + ColorChoice::Never => termcolor::ColorChoice::Never, + ColorChoice::Auto => termcolor::ColorChoice::Auto, + ColorChoice::Always => termcolor::ColorChoice::Always, + ColorChoice::Ansi => termcolor::ColorChoice::AlwaysAnsi, + } + } +} + +/// Indicates the line context options ripgrep should use for output. +/// +/// The default is no context at all. +#[derive(Debug, Eq, PartialEq)] +pub(crate) enum ContextMode { + /// All lines will be printed. That is, the context is unbounded. + Passthru, + /// Only show a certain number of lines before and after each match. + Limited(ContextModeLimited), +} + +impl Default for ContextMode { + fn default() -> ContextMode { + ContextMode::Limited(ContextModeLimited::default()) + } +} + +impl ContextMode { + /// Set the "before" context. + /// + /// If this was set to "passthru" context, then it is overridden in favor + /// of limited context with the given value for "before" and `0` for + /// "after." + pub(crate) fn set_before(&mut self, lines: usize) { + match *self { + ContextMode::Passthru => { + *self = ContextMode::Limited(ContextModeLimited { + before: Some(lines), + after: None, + both: None, + }) + } + ContextMode::Limited(ContextModeLimited { + ref mut before, + .. + }) => *before = Some(lines), + } + } + + /// Set the "after" context. + /// + /// If this was set to "passthru" context, then it is overridden in favor + /// of limited context with the given value for "after" and `0` for + /// "before." + pub(crate) fn set_after(&mut self, lines: usize) { + match *self { + ContextMode::Passthru => { + *self = ContextMode::Limited(ContextModeLimited { + before: None, + after: Some(lines), + both: None, + }) + } + ContextMode::Limited(ContextModeLimited { + ref mut after, .. + }) => *after = Some(lines), + } + } + + /// Set the "both" context. + /// + /// If this was set to "passthru" context, then it is overridden in favor + /// of limited context with the given value for "both" and `None` for + /// "before" and "after". + pub(crate) fn set_both(&mut self, lines: usize) { + match *self { + ContextMode::Passthru => { + *self = ContextMode::Limited(ContextModeLimited { + before: None, + after: None, + both: Some(lines), + }) + } + ContextMode::Limited(ContextModeLimited { + ref mut both, .. + }) => *both = Some(lines), + } + } + + /// A convenience function for use in tests that returns the limited + /// context. If this mode isn't limited, then it panics. + #[cfg(test)] + pub(crate) fn get_limited(&self) -> (usize, usize) { + match *self { + ContextMode::Passthru => unreachable!("context mode is passthru"), + ContextMode::Limited(ref limited) => limited.get(), + } + } +} + +/// A context mode for a finite number of lines. +/// +/// Namely, this indicates that a specific number of lines (possibly zero) +/// should be shown before and/or after each matching line. +/// +/// Note that there is a subtle difference between `Some(0)` and `None`. In the +/// former case, it happens when `0` is given explicitly, where as `None` is +/// the default value and occurs when no value is specified. +/// +/// `both` is only set by the -C/--context flag. The reason why we don't just +/// set before = after = --context is because the before and after context +/// settings always take precedent over the -C/--context setting, regardless of +/// order. Thus, we need to keep track of them separately. +#[derive(Debug, Default, Eq, PartialEq)] +pub(crate) struct ContextModeLimited { + before: Option, + after: Option, + both: Option, +} + +impl ContextModeLimited { + /// Returns the specific number of contextual lines that should be shown + /// around each match. This takes proper precedent into account, i.e., + /// that `before` and `after` both partially override `both` in all cases. + /// + /// By default, this returns `(0, 0)`. + pub(crate) fn get(&self) -> (usize, usize) { + let (mut before, mut after) = + self.both.map(|lines| (lines, lines)).unwrap_or((0, 0)); + // --before and --after always override --context, regardless + // of where they appear relative to each other. + if let Some(lines) = self.before { + before = lines; + } + if let Some(lines) = self.after { + after = lines; + } + (before, after) + } +} + +/// Represents the separator to use between non-contiguous sections of +/// contextual lines. +/// +/// The default is `--`. +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct ContextSeparator(Option); + +impl Default for ContextSeparator { + fn default() -> ContextSeparator { + ContextSeparator(Some(BString::from("--"))) + } +} + +impl ContextSeparator { + /// Create a new context separator from the user provided argument. This + /// handles unescaping. + pub(crate) fn new(os: &OsStr) -> anyhow::Result { + let Some(string) = os.to_str() else { + anyhow::bail!( + "separator must be valid UTF-8 (use escape sequences \ + to provide a separator that is not valid UTF-8)" + ) + }; + Ok(ContextSeparator(Some(Vec::unescape_bytes(string).into()))) + } + + /// Creates a new separator that instructs the printer to disable contextual + /// separators entirely. + pub(crate) fn disabled() -> ContextSeparator { + ContextSeparator(None) + } + + /// Return the raw bytes of this separator. + /// + /// If context separators were disabled, then this returns `None`. + /// + /// Note that this may return a `Some` variant with zero bytes. + pub(crate) fn into_bytes(self) -> Option> { + self.0.map(|sep| sep.into()) + } +} + +/// The encoding mode the searcher will use. +/// +/// The default is `Auto`. +#[derive(Debug, Default, Eq, PartialEq)] +pub(crate) enum EncodingMode { + /// Use only BOM sniffing to auto-detect an encoding. + #[default] + Auto, + /// Use an explicit encoding forcefully, but let BOM sniffing override it. + Some(grep::searcher::Encoding), + /// Use no explicit encoding and disable all BOM sniffing. This will + /// always result in searching the raw bytes, regardless of their + /// true encoding. + Disabled, +} + +/// The regex engine to use. +/// +/// The default is `Default`. +#[derive(Debug, Default, Eq, PartialEq)] +pub(crate) enum EngineChoice { + /// Uses the default regex engine: Rust's `regex` crate. + /// + /// (Well, technically it uses `regex-automata`, but `regex-automata` is + /// the implementation of the `regex` crate.) + #[default] + Default, + /// Dynamically select the right engine to use. + /// + /// This works by trying to use the default engine, and if the pattern does + /// not compile, it switches over to the PCRE2 engine if it's available. + Auto, + /// Uses the PCRE2 regex engine if it's available. + PCRE2, +} + +/// The field context separator to use to between metadata for each contextual +/// line. +/// +/// The default is `-`. +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct FieldContextSeparator(BString); + +impl Default for FieldContextSeparator { + fn default() -> FieldContextSeparator { + FieldContextSeparator(BString::from("-")) + } +} + +impl FieldContextSeparator { + /// Create a new separator from the given argument value provided by the + /// user. Unescaping it automatically handled. + pub(crate) fn new(os: &OsStr) -> anyhow::Result { + let Some(string) = os.to_str() else { + anyhow::bail!( + "separator must be valid UTF-8 (use escape sequences \ + to provide a separator that is not valid UTF-8)" + ) + }; + Ok(FieldContextSeparator(Vec::unescape_bytes(string).into())) + } + + /// Return the raw bytes of this separator. + /// + /// Note that this may return an empty `Vec`. + pub(crate) fn into_bytes(self) -> Vec { + self.0.into() + } +} + +/// The field match separator to use to between metadata for each matching +/// line. +/// +/// The default is `:`. +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct FieldMatchSeparator(BString); + +impl Default for FieldMatchSeparator { + fn default() -> FieldMatchSeparator { + FieldMatchSeparator(BString::from(":")) + } +} + +impl FieldMatchSeparator { + /// Create a new separator from the given argument value provided by the + /// user. Unescaping it automatically handled. + pub(crate) fn new(os: &OsStr) -> anyhow::Result { + let Some(string) = os.to_str() else { + anyhow::bail!( + "separator must be valid UTF-8 (use escape sequences \ + to provide a separator that is not valid UTF-8)" + ) + }; + Ok(FieldMatchSeparator(Vec::unescape_bytes(string).into())) + } + + /// Return the raw bytes of this separator. + /// + /// Note that this may return an empty `Vec`. + pub(crate) fn into_bytes(self) -> Vec { + self.0.into() + } +} + +/// The type of logging to do. `Debug` emits some details while `Trace` emits +/// much more. +#[derive(Debug, Eq, PartialEq)] +pub(crate) enum LoggingMode { + Debug, + Trace, +} + +/// Indicates when to use memory maps. +/// +/// The default is `Auto`. +#[derive(Debug, Default, Eq, PartialEq)] +pub(crate) enum MmapMode { + /// This instructs ripgrep to use heuristics for selecting when to and not + /// to use memory maps for searching. + #[default] + Auto, + /// This instructs ripgrep to always try memory maps when possible. (Memory + /// maps are not possible to use in all circumstances, for example, for + /// virtual files.) + AlwaysTryMmap, + /// Never use memory maps under any circumstances. This includes even + /// when multi-line search is enabled where ripgrep will read the entire + /// contents of a file on to the heap before searching it. + Never, +} + +/// Represents a source of patterns that ripgrep should search for. +/// +/// The reason to unify these is so that we can retain the order of `-f/--flag` +/// and `-e/--regexp` flags relative to one another. +#[derive(Debug, Eq, PartialEq)] +pub(crate) enum PatternSource { + /// Comes from the `-e/--regexp` flag. + Regexp(String), + /// Comes from the `-f/--file` flag. + File(PathBuf), +} + +/// The sort criteria, if present. +#[derive(Debug, Eq, PartialEq)] +pub(crate) struct SortMode { + /// Whether to reverse the sort criteria (i.e., descending order). + pub(crate) reverse: bool, + /// The actual sorting criteria. + pub(crate) kind: SortModeKind, +} + +/// The criteria to use for sorting. +#[derive(Debug, Eq, PartialEq)] +pub(crate) enum SortModeKind { + /// Sort by path. + Path, + /// Sort by last modified time. + LastModified, + /// Sort by last accessed time. + LastAccessed, + /// Sort by creation time. + Created, +} + +impl SortMode { + /// Checks whether the selected sort mode is supported. If it isn't, an + /// error (hopefully explaining why) is returned. + pub(crate) fn supported(&self) -> anyhow::Result<()> { + match self.kind { + SortModeKind::Path => Ok(()), + SortModeKind::LastModified => { + let md = std::env::current_exe() + .and_then(|p| p.metadata()) + .and_then(|md| md.modified()); + let Err(err) = md else { return Ok(()) }; + anyhow::bail!( + "sorting by last modified isn't supported: {err}" + ); + } + SortModeKind::LastAccessed => { + let md = std::env::current_exe() + .and_then(|p| p.metadata()) + .and_then(|md| md.accessed()); + let Err(err) = md else { return Ok(()) }; + anyhow::bail!( + "sorting by last accessed isn't supported: {err}" + ); + } + SortModeKind::Created => { + let md = std::env::current_exe() + .and_then(|p| p.metadata()) + .and_then(|md| md.created()); + let Err(err) = md else { return Ok(()) }; + anyhow::bail!( + "sorting by creation time isn't supported: {err}" + ); + } + } + } +} + +/// A single instance of either a change or a selection of one ripgrep's +/// file types. +#[derive(Debug, Eq, PartialEq)] +pub(crate) enum TypeChange { + /// Clear the given type from ripgrep. + Clear { name: String }, + /// Add the given type definition (name and glob) to ripgrep. + Add { def: String }, + /// Select the given type for filtering. + Select { name: String }, + /// Select the given type for filtering but negate it. + Negate { name: String }, +} diff --git a/userland/ripgrep/crates/core/flags/mod.rs b/userland/ripgrep/crates/core/flags/mod.rs new file mode 100644 index 0000000..7bf46db --- /dev/null +++ b/userland/ripgrep/crates/core/flags/mod.rs @@ -0,0 +1,302 @@ +/*! +Defines ripgrep's command line interface. + +This modules deals with everything involving ripgrep's flags and positional +arguments. This includes generating shell completions, `--help` output and even +ripgrep's man page. It's also responsible for parsing and validating every +flag (including reading ripgrep's config file), and manages the contact points +between these flags and ripgrep's cast of supporting libraries. For example, +once [`HiArgs`] has been created, it knows how to create a multi threaded +recursive directory traverser. +*/ +use std::{ + ffi::OsString, + fmt::Debug, + panic::{RefUnwindSafe, UnwindSafe}, +}; + +pub(crate) use crate::flags::{ + complete::{ + bash::generate as generate_complete_bash, + fish::generate as generate_complete_fish, + powershell::generate as generate_complete_powershell, + zsh::generate as generate_complete_zsh, + }, + doc::{ + help::{ + generate_long as generate_help_long, + generate_short as generate_help_short, + }, + man::generate as generate_man_page, + version::{ + generate_long as generate_version_long, + generate_pcre2 as generate_version_pcre2, + generate_short as generate_version_short, + }, + }, + hiargs::HiArgs, + lowargs::{GenerateMode, Mode, SearchMode, SpecialMode}, + parse::{ParseResult, parse}, +}; + +mod complete; +mod config; +mod defs; +mod doc; +mod hiargs; +mod lowargs; +mod parse; + +/// A trait that encapsulates the definition of an optional flag for ripgrep. +/// +/// This trait is meant to be used via dynamic dispatch. Namely, the `defs` +/// module provides a single global slice of `&dyn Flag` values correspondings +/// to all of the flags in ripgrep. +/// +/// ripgrep's required positional arguments are handled by the parser and by +/// the conversion from low-level arguments to high level arguments. Namely, +/// all of ripgrep's positional arguments are treated as file paths, except +/// in certain circumstances where the first argument is treated as a regex +/// pattern. +/// +/// Note that each implementation of this trait requires a long flag name, +/// but can also optionally have a short version and even a negation flag. +/// For example, the `-E/--encoding` flag accepts a value, but it also has a +/// `--no-encoding` negation flag for reverting back to "automatic" encoding +/// detection. All three of `-E`, `--encoding` and `--no-encoding` are provided +/// by a single implementation of this trait. +/// +/// ripgrep only supports flags that are switches or flags that accept a single +/// value. Flags that accept multiple values are an unsupported aberration. +trait Flag: Debug + Send + Sync + UnwindSafe + RefUnwindSafe + 'static { + /// Returns true if this flag is a switch. When a flag is a switch, the + /// CLI parser will not look for a value after the flag is seen. + fn is_switch(&self) -> bool; + + /// A short single byte name for this flag. This returns `None` by default, + /// which signifies that the flag has no short name. + /// + /// The byte returned must be an ASCII codepoint that is a `.` or is + /// alpha-numeric. + fn name_short(&self) -> Option { + None + } + + /// Returns the long name of this flag. All flags must have a "long" name. + /// + /// The long name must be at least 2 bytes, and all of its bytes must be + /// ASCII codepoints that are either `-` or alpha-numeric. + fn name_long(&self) -> &'static str; + + /// Returns a list of aliases for this flag. + /// + /// The aliases must follow the same rules as `Flag::name_long`. + /// + /// By default, an empty slice is returned. + fn aliases(&self) -> &'static [&'static str] { + &[] + } + + /// Returns a negated name for this flag. The negation of a flag is + /// intended to have the opposite meaning of a flag or to otherwise turn + /// something "off" or revert it to its default behavior. + /// + /// Negated flags are not listed in their own section in the `-h/--help` + /// output or man page. Instead, they are automatically mentioned at the + /// end of the documentation section of the flag they negated. + /// + /// The aliases must follow the same rules as `Flag::name_long`. + /// + /// By default, a flag has no negation and this returns `None`. + fn name_negated(&self) -> Option<&'static str> { + None + } + + /// Returns the variable name describing the type of value this flag + /// accepts. This should always be set for non-switch flags and never set + /// for switch flags. + /// + /// For example, the `--max-count` flag has its variable name set to `NUM`. + /// + /// The convention is to capitalize variable names. + /// + /// By default this returns `None`. + fn doc_variable(&self) -> Option<&'static str> { + None + } + + /// Returns the category of this flag. + /// + /// Every flag must have a single category. Categories are used to organize + /// flags in the generated documentation. + fn doc_category(&self) -> Category; + + /// A (very) short documentation string describing what this flag does. + /// + /// This may sacrifice "proper English" in order to be as terse as + /// possible. Generally, we try to ensure that `rg -h` doesn't have any + /// lines that exceed 79 columns. + fn doc_short(&self) -> &'static str; + + /// A (possibly very) longer documentation string describing in full + /// detail what this flag does. This should be in mandoc/mdoc format. + fn doc_long(&self) -> &'static str; + + /// If this is a non-switch flag that accepts a small set of specific + /// values, then this should list them. + /// + /// This returns an empty slice by default. + fn doc_choices(&self) -> &'static [&'static str] { + &[] + } + + fn completion_type(&self) -> CompletionType { + CompletionType::Other + } + + /// Given the parsed value (which might just be a switch), this should + /// update the state in `args` based on the value given for this flag. + /// + /// This may update state for other flags as appropriate. + /// + /// The `-V/--version` and `-h/--help` flags are treated specially in the + /// parser and should do nothing here. + /// + /// By convention, implementations should generally not try to "do" + /// anything other than validate the value given. For example, the + /// implementation for `--hostname-bin` should not try to resolve the + /// hostname to use by running the binary provided. That should be saved + /// for a later step. This convention is used to ensure that getting the + /// low-level arguments is as reliable and quick as possible. It also + /// ensures that "doing something" occurs a minimal number of times. For + /// example, by avoiding trying to find the hostname here, we can do it + /// once later no matter how many times `--hostname-bin` is provided. + /// + /// Implementations should not include the flag name in the error message + /// returned. The flag name is included automatically by the parser. + fn update( + &self, + value: FlagValue, + args: &mut crate::flags::lowargs::LowArgs, + ) -> anyhow::Result<()>; +} + +/// The category that a flag belongs to. +/// +/// Categories are used to organize flags into "logical" groups in the +/// generated documentation. +#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq, PartialOrd, Ord)] +enum Category { + /// Flags related to how ripgrep reads its input. Its "input" generally + /// consists of the patterns it is trying to match and the haystacks it is + /// trying to search. + Input, + /// Flags related to the operation of the search itself. For example, + /// whether case insensitive matching is enabled. + Search, + /// Flags related to how ripgrep filters haystacks. For example, whether + /// to respect gitignore files or not. + Filter, + /// Flags related to how ripgrep shows its search results. For example, + /// whether to show line numbers or not. + Output, + /// Flags related to changing ripgrep's output at a more fundamental level. + /// For example, flags like `--count` suppress printing of individual + /// lines, and instead just print the total count of matches for each file + /// searched. + OutputModes, + /// Flags related to logging behavior such as emitting non-fatal error + /// messages or printing search statistics. + Logging, + /// Other behaviors not related to ripgrep's core functionality. For + /// example, printing the file type globbing rules, or printing the list + /// of files ripgrep would search without actually searching them. + OtherBehaviors, +} + +impl Category { + /// Returns a string representation of this category. + /// + /// This string is the name of the variable used in various templates for + /// generated documentation. This name can be used for interpolation. + fn as_str(&self) -> &'static str { + match *self { + Category::Input => "input", + Category::Search => "search", + Category::Filter => "filter", + Category::Output => "output", + Category::OutputModes => "output-modes", + Category::Logging => "logging", + Category::OtherBehaviors => "other-behaviors", + } + } +} + +/// The kind of argument a flag accepts, to be used for shell completions. +#[derive(Clone, Copy, Debug)] +enum CompletionType { + /// No special category. is_switch() and doc_choices() may apply. + Other, + /// A path to a file. + Filename, + /// A command in $PATH. + Executable, + /// The name of a file type, as used by e.g. --type. + Filetype, + /// The name of an encoding_rs encoding, as used by --encoding. + Encoding, +} + +/// Represents a value parsed from the command line. +/// +/// This doesn't include the corresponding flag, but values come in one of +/// two forms: a switch (on or off) or an arbitrary value. +/// +/// Note that the CLI doesn't directly support negated switches. For example, +/// you can'd do anything like `-n=false` or any of that nonsense. Instead, +/// the CLI parser knows about which flag names are negations and which aren't +/// (courtesy of the `Flag` trait). If a flag given is known as a negation, +/// then a `FlagValue::Switch(false)` value is passed into `Flag::update`. +#[derive(Debug)] +enum FlagValue { + /// A flag that is either on or off. + Switch(bool), + /// A flag that comes with an arbitrary user value. + Value(OsString), +} + +impl FlagValue { + /// Return the yes or no value of this switch. + /// + /// If this flag value is not a switch, then this panics. + /// + /// This is useful when writing the implementation of `Flag::update`. + /// namely, callers usually know whether a switch or a value is expected. + /// If a flag is something different, then it indicates a bug, and thus a + /// panic is acceptable. + fn unwrap_switch(self) -> bool { + match self { + FlagValue::Switch(yes) => yes, + FlagValue::Value(_) => { + unreachable!("got flag value but expected switch") + } + } + } + + /// Return the user provided value of this flag. + /// + /// If this flag is a switch, then this panics. + /// + /// This is useful when writing the implementation of `Flag::update`. + /// namely, callers usually know whether a switch or a value is expected. + /// If a flag is something different, then it indicates a bug, and thus a + /// panic is acceptable. + fn unwrap_value(self) -> OsString { + match self { + FlagValue::Switch(_) => { + unreachable!("got switch but expected flag value") + } + FlagValue::Value(v) => v, + } + } +} diff --git a/userland/ripgrep/crates/core/flags/parse.rs b/userland/ripgrep/crates/core/flags/parse.rs new file mode 100644 index 0000000..edda40a --- /dev/null +++ b/userland/ripgrep/crates/core/flags/parse.rs @@ -0,0 +1,476 @@ +/*! +Parses command line arguments into a structured and typed representation. +*/ + +use std::{borrow::Cow, collections::BTreeSet, ffi::OsString}; + +use anyhow::Context; + +use crate::flags::{ + Flag, FlagValue, + defs::FLAGS, + hiargs::HiArgs, + lowargs::{LoggingMode, LowArgs, SpecialMode}, +}; + +/// The result of parsing CLI arguments. +/// +/// This is basically a `anyhow::Result`, but with one extra variant that is +/// inhabited whenever ripgrep should execute a "special" mode. That is, when a +/// user provides the `-h/--help` or `-V/--version` flags. +/// +/// This special variant exists to allow CLI parsing to short circuit as +/// quickly as is reasonable. For example, it lets CLI parsing avoid reading +/// ripgrep's configuration and converting low level arguments into a higher +/// level representation. +#[derive(Debug)] +pub(crate) enum ParseResult { + Special(SpecialMode), + Ok(T), + Err(anyhow::Error), +} + +impl ParseResult { + /// If this result is `Ok`, then apply `then` to it. Otherwise, return this + /// result unchanged. + fn and_then( + self, + mut then: impl FnMut(T) -> ParseResult, + ) -> ParseResult { + match self { + ParseResult::Special(mode) => ParseResult::Special(mode), + ParseResult::Ok(t) => then(t), + ParseResult::Err(err) => ParseResult::Err(err), + } + } +} + +/// Parse CLI arguments and convert then to their high level representation. +pub(crate) fn parse() -> ParseResult { + parse_low().and_then(|low| match HiArgs::from_low_args(low) { + Ok(hi) => ParseResult::Ok(hi), + Err(err) => ParseResult::Err(err), + }) +} + +/// Parse CLI arguments only into their low level representation. +/// +/// This takes configuration into account. That is, it will try to read +/// `RIPGREP_CONFIG_PATH` and prepend any arguments found there to the +/// arguments passed to this process. +/// +/// This will also set one-time global state flags, such as the log level and +/// whether messages should be printed. +fn parse_low() -> ParseResult { + if let Err(err) = crate::logger::Logger::init() { + let err = anyhow::anyhow!("failed to initialize logger: {err}"); + return ParseResult::Err(err); + } + + let parser = Parser::new(); + let mut low = LowArgs::default(); + if let Err(err) = parser.parse(std::env::args_os().skip(1), &mut low) { + return ParseResult::Err(err); + } + // Even though we haven't parsed the config file yet (assuming it exists), + // we can still use the arguments given on the CLI to setup ripgrep's + // logging preferences. Even if the config file changes them in some way, + // it's really the best we can do. This way, for example, folks can pass + // `--trace` and see any messages logged during config file parsing. + set_log_levels(&low); + // Before we try to take configuration into account, we can bail early + // if a special mode was enabled. This is basically only for version and + // help output which shouldn't be impacted by extra configuration. + if let Some(special) = low.special.take() { + return ParseResult::Special(special); + } + // If the end user says no config, then respect it. + if low.no_config { + log::debug!("not reading config files because --no-config is present"); + return ParseResult::Ok(low); + } + // Look for arguments from a config file. If we got nothing (whether the + // file is empty or RIPGREP_CONFIG_PATH wasn't set), then we don't need + // to re-parse. + let config_args = crate::flags::config::args(); + if config_args.is_empty() { + log::debug!("no extra arguments found from configuration file"); + return ParseResult::Ok(low); + } + // The final arguments are just the arguments from the CLI appending to + // the end of the config arguments. + let mut final_args = config_args; + final_args.extend(std::env::args_os().skip(1)); + + // Now do the CLI parsing dance again. + let mut low = LowArgs::default(); + if let Err(err) = parser.parse(final_args.into_iter(), &mut low) { + return ParseResult::Err(err); + } + // Reset the message and logging levels, since they could have changed. + set_log_levels(&low); + ParseResult::Ok(low) +} + +/// Sets global state flags that control logging based on low-level arguments. +fn set_log_levels(low: &LowArgs) { + crate::messages::set_messages(!low.no_messages); + crate::messages::set_ignore_messages(!low.no_ignore_messages); + match low.logging { + Some(LoggingMode::Trace) => { + log::set_max_level(log::LevelFilter::Trace) + } + Some(LoggingMode::Debug) => { + log::set_max_level(log::LevelFilter::Debug) + } + None => log::set_max_level(log::LevelFilter::Warn), + } +} + +/// Parse the sequence of CLI arguments given a low level typed set of +/// arguments. +/// +/// This is exposed for testing that the correct low-level arguments are parsed +/// from a CLI. It just runs the parser once over the CLI arguments. It doesn't +/// setup logging or read from a config file. +/// +/// This assumes the iterator given does *not* begin with the binary name. +#[cfg(test)] +pub(crate) fn parse_low_raw( + rawargs: impl IntoIterator>, +) -> anyhow::Result { + let mut args = LowArgs::default(); + Parser::new().parse(rawargs, &mut args)?; + Ok(args) +} + +/// Return the metadata for the flag of the given name. +pub(super) fn lookup(name: &str) -> Option<&'static dyn Flag> { + // N.B. Creating a new parser might look expensive, but it only builds + // the lookup trie exactly once. That is, we get a `&'static Parser` from + // `Parser::new()`. + match Parser::new().find_long(name) { + FlagLookup::Match(&FlagInfo { flag, .. }) => Some(flag), + _ => None, + } +} + +/// A parser for turning a sequence of command line arguments into a more +/// strictly typed set of arguments. +#[derive(Debug)] +struct Parser { + /// A single map that contains all possible flag names. This includes + /// short and long names, aliases and negations. This maps those names to + /// indices into `info`. + map: FlagMap, + /// A map from IDs returned by the `map` to the corresponding flag + /// information. + info: Vec, +} + +impl Parser { + /// Create a new parser. + /// + /// This always creates the same parser and only does it once. Callers may + /// call this repeatedly, and the parser will only be built once. + fn new() -> &'static Parser { + use std::sync::OnceLock; + + // Since a parser's state is immutable and completely determined by + // FLAGS, and since FLAGS is a constant, we can initialize it exactly + // once. + static P: OnceLock = OnceLock::new(); + P.get_or_init(|| { + let mut infos = vec![]; + for &flag in FLAGS.iter() { + infos.push(FlagInfo { + flag, + name: Ok(flag.name_long()), + kind: FlagInfoKind::Standard, + }); + for alias in flag.aliases() { + infos.push(FlagInfo { + flag, + name: Ok(alias), + kind: FlagInfoKind::Alias, + }); + } + if let Some(byte) = flag.name_short() { + infos.push(FlagInfo { + flag, + name: Err(byte), + kind: FlagInfoKind::Standard, + }); + } + if let Some(name) = flag.name_negated() { + infos.push(FlagInfo { + flag, + name: Ok(name), + kind: FlagInfoKind::Negated, + }); + } + } + let map = FlagMap::new(&infos); + Parser { map, info: infos } + }) + } + + /// Parse the given CLI arguments into a low level representation. + /// + /// The iterator given should *not* start with the binary name. + fn parse(&self, rawargs: I, args: &mut LowArgs) -> anyhow::Result<()> + where + I: IntoIterator, + O: Into, + { + let mut p = lexopt::Parser::from_args(rawargs); + while let Some(arg) = p.next().context("invalid CLI arguments")? { + let lookup = match arg { + lexopt::Arg::Value(value) => { + args.positional.push(value); + continue; + } + lexopt::Arg::Short(ch) if ch == 'h' => { + // Special case -h/--help since behavior is different + // based on whether short or long flag is given. + args.special = Some(SpecialMode::HelpShort); + continue; + } + lexopt::Arg::Short(ch) if ch == 'V' => { + // Special case -V/--version since behavior is different + // based on whether short or long flag is given. + args.special = Some(SpecialMode::VersionShort); + continue; + } + lexopt::Arg::Short(ch) => self.find_short(ch), + lexopt::Arg::Long(name) if name == "help" => { + // Special case -h/--help since behavior is different + // based on whether short or long flag is given. + args.special = Some(SpecialMode::HelpLong); + continue; + } + lexopt::Arg::Long(name) if name == "version" => { + // Special case -V/--version since behavior is different + // based on whether short or long flag is given. + args.special = Some(SpecialMode::VersionLong); + continue; + } + lexopt::Arg::Long(name) => self.find_long(name), + }; + let mat = match lookup { + FlagLookup::Match(mat) => mat, + FlagLookup::UnrecognizedShort(name) => { + anyhow::bail!("unrecognized flag -{name}") + } + FlagLookup::UnrecognizedLong(name) => { + let mut msg = format!("unrecognized flag --{name}"); + if let Some(suggest_msg) = suggest(&name) { + msg = format!("{msg}\n\n{suggest_msg}"); + } + anyhow::bail!("{msg}") + } + }; + let value = if matches!(mat.kind, FlagInfoKind::Negated) { + // Negated flags are always switches, even if the non-negated + // flag is not. For example, --context-separator accepts a + // value, but --no-context-separator does not. + FlagValue::Switch(false) + } else if mat.flag.is_switch() { + FlagValue::Switch(true) + } else { + FlagValue::Value(p.value().with_context(|| { + format!("missing value for flag {mat}") + })?) + }; + mat.flag + .update(value, args) + .with_context(|| format!("error parsing flag {mat}"))?; + } + Ok(()) + } + + /// Look for a flag by its short name. + fn find_short(&self, ch: char) -> FlagLookup<'_> { + if !ch.is_ascii() { + return FlagLookup::UnrecognizedShort(ch); + } + let byte = u8::try_from(ch).unwrap(); + let Some(index) = self.map.find(&[byte]) else { + return FlagLookup::UnrecognizedShort(ch); + }; + FlagLookup::Match(&self.info[index]) + } + + /// Look for a flag by its long name. + /// + /// This also works for aliases and negated names. + fn find_long(&self, name: &str) -> FlagLookup<'_> { + let Some(index) = self.map.find(name.as_bytes()) else { + return FlagLookup::UnrecognizedLong(name.to_string()); + }; + FlagLookup::Match(&self.info[index]) + } +} + +/// The result of looking up a flag name. +#[derive(Debug)] +enum FlagLookup<'a> { + /// Lookup found a match and the metadata for the flag is attached. + Match(&'a FlagInfo), + /// The given short name is unrecognized. + UnrecognizedShort(char), + /// The given long name is unrecognized. + UnrecognizedLong(String), +} + +/// The info about a flag associated with a flag's ID in the flag map. +#[derive(Debug)] +struct FlagInfo { + /// The flag object and its associated metadata. + flag: &'static dyn Flag, + /// The actual name that is stored in the Aho-Corasick automaton. When this + /// is a byte, it corresponds to a short single character ASCII flag. The + /// actual pattern that's in the Aho-Corasick automaton is just the single + /// byte. + name: Result<&'static str, u8>, + /// The type of flag that is stored for the corresponding Aho-Corasick + /// pattern. + kind: FlagInfoKind, +} + +/// The kind of flag that is being matched. +#[derive(Debug)] +enum FlagInfoKind { + /// A standard flag, e.g., --passthru. + Standard, + /// A negation of a standard flag, e.g., --no-multiline. + Negated, + /// An alias for a standard flag, e.g., --passthrough. + Alias, +} + +impl std::fmt::Display for FlagInfo { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + match self.name { + Ok(long) => write!(f, "--{long}"), + Err(short) => write!(f, "-{short}", short = char::from(short)), + } + } +} + +/// A map from flag names (short, long, negated and aliases) to their ID. +/// +/// Once an ID is known, it can be used to look up a flag's metadata in the +/// parser's internal state. +#[derive(Debug)] +struct FlagMap { + map: std::collections::HashMap, usize>, +} + +impl FlagMap { + /// Create a new map of flags for the given flag information. + /// + /// The index of each flag info corresponds to its ID. + fn new(infos: &[FlagInfo]) -> FlagMap { + let mut map = std::collections::HashMap::with_capacity(infos.len()); + for (i, info) in infos.iter().enumerate() { + match info.name { + Ok(name) => { + assert_eq!(None, map.insert(name.as_bytes().to_vec(), i)); + } + Err(byte) => { + assert_eq!(None, map.insert(vec![byte], i)); + } + } + } + FlagMap { map } + } + + /// Look for a match of `name` in the given Aho-Corasick automaton. + /// + /// This only returns a match if the one found has a length equivalent to + /// the length of the name given. + fn find(&self, name: &[u8]) -> Option { + self.map.get(name).copied() + } +} + +/// Possibly return a message suggesting flags similar in the name to the one +/// given. +/// +/// The one given should be a flag given by the user (without the leading +/// dashes) that was unrecognized. This attempts to find existing flags that +/// are similar to the one given. +fn suggest(unrecognized: &str) -> Option { + let similars = find_similar_names(unrecognized); + if similars.is_empty() { + return None; + } + let list = similars + .into_iter() + .map(|name| format!("--{name}")) + .collect::>() + .join(", "); + Some(format!("similar flags that are available: {list}")) +} + +/// Return a sequence of names similar to the unrecognized name given. +fn find_similar_names(unrecognized: &str) -> Vec<&'static str> { + // The jaccard similarity threshold at which we consider two flag names + // similar enough that it's worth suggesting it to the end user. + // + // This value was determined by some ad hoc experimentation. It might need + // further tweaking. + const THRESHOLD: f64 = 0.4; + + let mut similar = vec![]; + let bow_given = ngrams(unrecognized); + for &flag in FLAGS.iter() { + let name = flag.name_long(); + let bow = ngrams(name); + if jaccard_index(&bow_given, &bow) >= THRESHOLD { + similar.push(name); + } + if let Some(name) = flag.name_negated() { + let bow = ngrams(name); + if jaccard_index(&bow_given, &bow) >= THRESHOLD { + similar.push(name); + } + } + for name in flag.aliases() { + let bow = ngrams(name); + if jaccard_index(&bow_given, &bow) >= THRESHOLD { + similar.push(name); + } + } + } + similar +} + +/// A "bag of words" is a set of ngrams. +type BagOfWords<'a> = BTreeSet>; + +/// Returns the jaccard index (a measure of similarity) between sets of ngrams. +fn jaccard_index(ngrams1: &BagOfWords<'_>, ngrams2: &BagOfWords<'_>) -> f64 { + let union = u32::try_from(ngrams1.union(ngrams2).count()) + .expect("fewer than u32::MAX flags"); + let intersection = u32::try_from(ngrams1.intersection(ngrams2).count()) + .expect("fewer than u32::MAX flags"); + f64::from(intersection) / f64::from(union) +} + +/// Returns all 3-grams in the slice given. +/// +/// If the slice doesn't contain a 3-gram, then one is artificially created by +/// padding it out with a character that will never appear in a flag name. +fn ngrams(flag_name: &str) -> BagOfWords<'_> { + // We only allow ASCII flag names, so we can just use bytes. + let slice = flag_name.as_bytes(); + let seq: Vec> = match slice.len() { + 0 => vec![Cow::Owned(b"!!!".to_vec())], + 1 => vec![Cow::Owned(vec![slice[0], b'!', b'!'])], + 2 => vec![Cow::Owned(vec![slice[0], slice[1], b'!'])], + _ => slice.windows(3).map(Cow::Borrowed).collect(), + }; + BTreeSet::from_iter(seq) +} diff --git a/userland/ripgrep/crates/core/haystack.rs b/userland/ripgrep/crates/core/haystack.rs new file mode 100644 index 0000000..f88b6de --- /dev/null +++ b/userland/ripgrep/crates/core/haystack.rs @@ -0,0 +1,160 @@ +/*! +Defines a builder for haystacks. + +A "haystack" represents something we want to search. It encapsulates the logic +for whether a haystack ought to be searched or not, separate from the standard +ignore rules and other filtering logic. + +Effectively, a haystack wraps a directory entry and adds some light application +level logic around it. +*/ + +use std::path::Path; + +/// A builder for constructing things to search over. +#[derive(Clone, Debug)] +pub(crate) struct HaystackBuilder { + strip_dot_prefix: bool, +} + +impl HaystackBuilder { + /// Return a new haystack builder with a default configuration. + pub(crate) fn new() -> HaystackBuilder { + HaystackBuilder { strip_dot_prefix: false } + } + + /// Create a new haystack from a possibly missing directory entry. + /// + /// If the directory entry isn't present, then the corresponding error is + /// logged if messages have been configured. Otherwise, if the directory + /// entry is deemed searchable, then it is returned as a haystack. + pub(crate) fn build_from_result( + &self, + result: Result, + ) -> Option { + match result { + Ok(dent) => self.build(dent), + Err(err) => { + err_message!("{err}"); + None + } + } + } + + /// Create a new haystack using this builder's configuration. + /// + /// If a directory entry could not be created or should otherwise not be + /// searched, then this returns `None` after emitting any relevant log + /// messages. + fn build(&self, dent: ignore::DirEntry) -> Option { + let hay = Haystack { dent, strip_dot_prefix: self.strip_dot_prefix }; + if let Some(err) = hay.dent.error() { + ignore_message!("{err}"); + } + // If this entry was explicitly provided by an end user, then we always + // want to search it. + if hay.is_explicit() { + return Some(hay); + } + // At this point, we only want to search something if it's explicitly a + // file. This omits symlinks. (If ripgrep was configured to follow + // symlinks, then they have already been followed by the directory + // traversal.) + if hay.is_file() { + return Some(hay); + } + // We got nothing. Emit a debug message, but only if this isn't a + // directory. Otherwise, emitting messages for directories is just + // noisy. + if !hay.is_dir() { + log::debug!( + "ignoring {}: failed to pass haystack filter: \ + file type: {:?}, metadata: {:?}", + hay.dent.path().display(), + hay.dent.file_type(), + hay.dent.metadata() + ); + } + None + } + + /// When enabled, if the haystack's file path starts with `./` then it is + /// stripped. + /// + /// This is useful when implicitly searching the current working directory. + pub(crate) fn strip_dot_prefix( + &mut self, + yes: bool, + ) -> &mut HaystackBuilder { + self.strip_dot_prefix = yes; + self + } +} + +/// A haystack is a thing we want to search. +/// +/// Generally, a haystack is either a file or stdin. +#[derive(Clone, Debug)] +pub(crate) struct Haystack { + dent: ignore::DirEntry, + strip_dot_prefix: bool, +} + +impl Haystack { + /// Return the file path corresponding to this haystack. + /// + /// If this haystack corresponds to stdin, then a special `` path + /// is returned instead. + pub(crate) fn path(&self) -> &Path { + if self.strip_dot_prefix && self.dent.path().starts_with("./") { + self.dent.path().strip_prefix("./").unwrap() + } else { + self.dent.path() + } + } + + /// Returns true if and only if this entry corresponds to stdin. + pub(crate) fn is_stdin(&self) -> bool { + self.dent.is_stdin() + } + + /// Returns true if and only if this entry corresponds to a haystack to + /// search that was explicitly supplied by an end user. + /// + /// Generally, this corresponds to either stdin or an explicit file path + /// argument. e.g., in `rg foo some-file ./some-dir/`, `some-file` is + /// an explicit haystack, but, e.g., `./some-dir/some-other-file` is not. + /// + /// However, note that ripgrep does not see through shell globbing. e.g., + /// in `rg foo ./some-dir/*`, `./some-dir/some-other-file` will be treated + /// as an explicit haystack. + pub(crate) fn is_explicit(&self) -> bool { + // stdin is obvious. When an entry has a depth of 0, that means it + // was explicitly provided to our directory iterator, which means it + // was in turn explicitly provided by the end user. The !is_dir check + // means that we want to search files even if their symlinks, again, + // because they were explicitly provided. (And we never want to try + // to search a directory.) + self.is_stdin() || (self.dent.depth() == 0 && !self.is_dir()) + } + + /// Returns true if and only if this haystack points to a directory after + /// following symbolic links. + fn is_dir(&self) -> bool { + let ft = match self.dent.file_type() { + None => return false, + Some(ft) => ft, + }; + if ft.is_dir() { + return true; + } + // If this is a symlink, then we want to follow it to determine + // whether it's a directory or not. + self.dent.path_is_symlink() && self.dent.path().is_dir() + } + + /// Returns true if and only if this haystack points to a file. + fn is_file(&self) -> bool { + self.dent.file_type().map_or(false, |ft| ft.is_file()) + } +} diff --git a/userland/ripgrep/crates/core/logger.rs b/userland/ripgrep/crates/core/logger.rs new file mode 100644 index 0000000..ce4e0e9 --- /dev/null +++ b/userland/ripgrep/crates/core/logger.rs @@ -0,0 +1,72 @@ +/*! +Defines a super simple logger that works with the `log` crate. + +We don't do anything fancy. We just need basic log levels and the ability to +print to stderr. We therefore avoid bringing in extra dependencies just for +this functionality. +*/ + +use log::Log; + +/// The simplest possible logger that logs to stderr. +/// +/// This logger does no filtering. Instead, it relies on the `log` crates +/// filtering via its global max_level setting. +#[derive(Debug)] +pub(crate) struct Logger(()); + +/// A singleton used as the target for an implementation of the `Log` trait. +const LOGGER: &'static Logger = &Logger(()); + +impl Logger { + /// Create a new logger that logs to stderr and initialize it as the + /// global logger. If there was a problem setting the logger, then an + /// error is returned. + pub(crate) fn init() -> Result<(), log::SetLoggerError> { + log::set_logger(LOGGER) + } +} + +impl Log for Logger { + fn enabled(&self, _: &log::Metadata<'_>) -> bool { + // We set the log level via log::set_max_level, so we don't need to + // implement filtering here. + true + } + + fn log(&self, record: &log::Record<'_>) { + match (record.file(), record.line()) { + (Some(file), Some(line)) => { + eprintln_locked!( + "{}|{}|{}:{}: {}", + record.level(), + record.target(), + file, + line, + record.args() + ); + } + (Some(file), None) => { + eprintln_locked!( + "{}|{}|{}: {}", + record.level(), + record.target(), + file, + record.args() + ); + } + _ => { + eprintln_locked!( + "{}|{}: {}", + record.level(), + record.target(), + record.args() + ); + } + } + } + + fn flush(&self) { + // We use eprintln_locked! which is flushed on every call. + } +} diff --git a/userland/ripgrep/crates/core/main.rs b/userland/ripgrep/crates/core/main.rs new file mode 100644 index 0000000..c74e645 --- /dev/null +++ b/userland/ripgrep/crates/core/main.rs @@ -0,0 +1,483 @@ +/*! +The main entry point into ripgrep. +*/ + +use std::{io::Write, process::ExitCode}; + +use ignore::WalkState; + +use crate::flags::{HiArgs, SearchMode}; + +#[macro_use] +mod messages; + +mod flags; +mod haystack; +mod logger; +mod search; + +// Since Rust no longer uses jemalloc by default, ripgrep will, by default, +// use the system allocator. On Linux, this would normally be glibc's +// allocator, which is pretty good. In particular, ripgrep does not have a +// particularly allocation heavy workload, so there really isn't much +// difference (for ripgrep's purposes) between glibc's allocator and jemalloc. +// +// However, when ripgrep is built with musl, this means ripgrep will use musl's +// allocator, which appears to be substantially worse. (musl's goal is not to +// have the fastest version of everything. Its goal is to be small and amenable +// to static compilation.) Even though ripgrep isn't particularly allocation +// heavy, musl's allocator appears to slow down ripgrep quite a bit. Therefore, +// when building with musl, we use jemalloc. +// +// We don't unconditionally use jemalloc because it can be nice to use the +// system's default allocator by default. Moreover, jemalloc seems to increase +// compilation times by a bit. +// +// Moreover, we only do this on 64-bit systems since jemalloc doesn't support +// i686. +#[cfg(all(target_env = "musl", target_pointer_width = "64"))] +#[global_allocator] +static ALLOC: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc; + +/// Then, as it was, then again it will be. +fn main() -> ExitCode { + match run(flags::parse()) { + Ok(code) => code, + Err(err) => { + // Look for a broken pipe error. In this case, we generally want + // to exit "gracefully" with a success exit code. This matches + // existing Unix convention. We need to handle this explicitly + // since the Rust runtime doesn't ask for PIPE signals, and thus + // we get an I/O error instead. Traditional C Unix applications + // quit by getting a PIPE signal that they don't handle, and thus + // the unhandled signal causes the process to unceremoniously + // terminate. + for cause in err.chain() { + if let Some(ioerr) = cause.downcast_ref::() { + if ioerr.kind() == std::io::ErrorKind::BrokenPipe { + return ExitCode::from(0); + } + } + } + eprintln_locked!("{:#}", err); + ExitCode::from(2) + } + } +} + +/// The main entry point for ripgrep. +/// +/// The given parse result determines ripgrep's behavior. The parse +/// result should be the result of parsing CLI arguments in a low level +/// representation, and then followed by an attempt to convert them into a +/// higher level representation. The higher level representation has some nicer +/// abstractions, for example, instead of representing the `-g/--glob` flag +/// as a `Vec` (as in the low level representation), the globs are +/// converted into a single matcher. +fn run(result: crate::flags::ParseResult) -> anyhow::Result { + use crate::flags::{Mode, ParseResult}; + + let args = match result { + ParseResult::Err(err) => return Err(err), + ParseResult::Special(mode) => return special(mode), + ParseResult::Ok(args) => args, + }; + let matched = match args.mode() { + Mode::Search(_) if !args.matches_possible() => false, + Mode::Search(mode) if args.threads() == 1 => search(&args, mode)?, + Mode::Search(mode) => search_parallel(&args, mode)?, + Mode::Files if args.threads() == 1 => files(&args)?, + Mode::Files => files_parallel(&args)?, + Mode::Types => return types(&args), + Mode::Generate(mode) => return generate(mode), + }; + Ok(if matched && (args.quiet() || !messages::errored()) { + ExitCode::from(0) + } else if messages::errored() { + ExitCode::from(2) + } else { + ExitCode::from(1) + }) +} + +/// The top-level entry point for single-threaded search. +/// +/// This recursively steps through the file list (current directory by default) +/// and searches each file sequentially. +fn search(args: &HiArgs, mode: SearchMode) -> anyhow::Result { + let started_at = std::time::Instant::now(); + let haystack_builder = args.haystack_builder(); + let unsorted = args + .walk_builder()? + .build() + .filter_map(|result| haystack_builder.build_from_result(result)); + let haystacks = args.sort(unsorted); + + let mut matched = false; + let mut searched = false; + let mut stats = args.stats(); + let mut searcher = args.search_worker( + args.matcher()?, + args.searcher()?, + args.printer(mode, args.stdout()), + )?; + for haystack in haystacks { + searched = true; + let search_result = match searcher.search(&haystack) { + Ok(search_result) => search_result, + // A broken pipe means graceful termination. + Err(err) if err.kind() == std::io::ErrorKind::BrokenPipe => break, + Err(err) => { + err_message!("{}: {}", haystack.path().display(), err); + continue; + } + }; + matched = matched || search_result.has_match(); + if let Some(ref mut stats) = stats { + *stats += search_result.stats().unwrap(); + } + if matched && args.quit_after_match() { + break; + } + } + if args.has_implicit_path() && !searched { + eprint_nothing_searched(); + } + if let Some(ref stats) = stats { + let wtr = searcher.printer().get_mut(); + let _ = print_stats(mode, stats, started_at, wtr); + } + Ok(matched) +} + +/// The top-level entry point for multi-threaded search. +/// +/// The parallelism is itself achieved by the recursive directory traversal. +/// All we need to do is feed it a worker for performing a search on each file. +/// +/// Requesting a sorted output from ripgrep (such as with `--sort path`) will +/// automatically disable parallelism and hence sorting is not handled here. +fn search_parallel(args: &HiArgs, mode: SearchMode) -> anyhow::Result { + use std::sync::atomic::{AtomicBool, Ordering}; + + let started_at = std::time::Instant::now(); + let haystack_builder = args.haystack_builder(); + let bufwtr = args.buffer_writer(); + let stats = args.stats().map(std::sync::Mutex::new); + let matched = AtomicBool::new(false); + let searched = AtomicBool::new(false); + + let mut searcher = args.search_worker( + args.matcher()?, + args.searcher()?, + args.printer(mode, bufwtr.buffer()), + )?; + args.walk_builder()?.build_parallel().run(|| { + let bufwtr = &bufwtr; + let stats = &stats; + let matched = &matched; + let searched = &searched; + let haystack_builder = &haystack_builder; + let mut searcher = searcher.clone(); + + Box::new(move |result| { + let haystack = match haystack_builder.build_from_result(result) { + Some(haystack) => haystack, + None => return WalkState::Continue, + }; + searched.store(true, Ordering::SeqCst); + searcher.printer().get_mut().clear(); + let search_result = match searcher.search(&haystack) { + Ok(search_result) => search_result, + Err(err) => { + err_message!("{}: {}", haystack.path().display(), err); + return WalkState::Continue; + } + }; + if search_result.has_match() { + matched.store(true, Ordering::SeqCst); + } + if let Some(ref locked_stats) = *stats { + let mut stats = locked_stats.lock().unwrap(); + *stats += search_result.stats().unwrap(); + } + if let Err(err) = bufwtr.print(searcher.printer().get_mut()) { + // A broken pipe means graceful termination. + if err.kind() == std::io::ErrorKind::BrokenPipe { + return WalkState::Quit; + } + // Otherwise, we continue on our merry way. + err_message!("{}: {}", haystack.path().display(), err); + } + if matched.load(Ordering::SeqCst) && args.quit_after_match() { + WalkState::Quit + } else { + WalkState::Continue + } + }) + }); + if args.has_implicit_path() && !searched.load(Ordering::SeqCst) { + eprint_nothing_searched(); + } + if let Some(ref locked_stats) = stats { + let stats = locked_stats.lock().unwrap(); + let mut wtr = searcher.printer().get_mut(); + let _ = print_stats(mode, &stats, started_at, &mut wtr); + let _ = bufwtr.print(&mut wtr); + } + Ok(matched.load(Ordering::SeqCst)) +} + +/// The top-level entry point for file listing without searching. +/// +/// This recursively steps through the file list (current directory by default) +/// and prints each path sequentially using a single thread. +fn files(args: &HiArgs) -> anyhow::Result { + let haystack_builder = args.haystack_builder(); + let unsorted = args + .walk_builder()? + .build() + .filter_map(|result| haystack_builder.build_from_result(result)); + let haystacks = args.sort(unsorted); + + let mut matched = false; + let mut path_printer = args.path_printer_builder().build(args.stdout()); + for haystack in haystacks { + matched = true; + if args.quit_after_match() { + break; + } + if let Err(err) = path_printer.write(haystack.path()) { + // A broken pipe means graceful termination. + if err.kind() == std::io::ErrorKind::BrokenPipe { + break; + } + // Otherwise, we have some other error that's preventing us from + // writing to stdout, so we should bubble it up. + return Err(err.into()); + } + } + Ok(matched) +} + +/// The top-level entry point for multi-threaded file listing without +/// searching. +/// +/// This recursively steps through the file list (current directory by default) +/// and prints each path sequentially using multiple threads. +/// +/// Requesting a sorted output from ripgrep (such as with `--sort path`) will +/// automatically disable parallelism and hence sorting is not handled here. +fn files_parallel(args: &HiArgs) -> anyhow::Result { + use std::{ + sync::{ + atomic::{AtomicBool, Ordering}, + mpsc, + }, + thread, + }; + + let haystack_builder = args.haystack_builder(); + let mut path_printer = args.path_printer_builder().build(args.stdout()); + let matched = AtomicBool::new(false); + let (tx, rx) = mpsc::channel::(); + + // We spawn a single printing thread to make sure we don't tear writes. + // We use a channel here under the presumption that it's probably faster + // than using a mutex in the worker threads below, but this has never been + // seriously litigated. + let print_thread = thread::spawn(move || -> std::io::Result<()> { + for haystack in rx.iter() { + path_printer.write(haystack.path())?; + } + Ok(()) + }); + args.walk_builder()?.build_parallel().run(|| { + let haystack_builder = &haystack_builder; + let matched = &matched; + let tx = tx.clone(); + + Box::new(move |result| { + let haystack = match haystack_builder.build_from_result(result) { + Some(haystack) => haystack, + None => return WalkState::Continue, + }; + matched.store(true, Ordering::SeqCst); + if args.quit_after_match() { + WalkState::Quit + } else { + match tx.send(haystack) { + Ok(_) => WalkState::Continue, + Err(_) => WalkState::Quit, + } + } + }) + }); + drop(tx); + if let Err(err) = print_thread.join().unwrap() { + // A broken pipe means graceful termination, so fall through. + // Otherwise, something bad happened while writing to stdout, so bubble + // it up. + if err.kind() != std::io::ErrorKind::BrokenPipe { + return Err(err.into()); + } + } + Ok(matched.load(Ordering::SeqCst)) +} + +/// The top-level entry point for `--type-list`. +fn types(args: &HiArgs) -> anyhow::Result { + let mut count = 0; + let mut stdout = args.stdout(); + for def in args.types().definitions() { + count += 1; + stdout.write_all(def.name().as_bytes())?; + stdout.write_all(b": ")?; + + let mut first = true; + for glob in def.globs() { + if !first { + stdout.write_all(b", ")?; + } + stdout.write_all(glob.as_bytes())?; + first = false; + } + stdout.write_all(b"\n")?; + } + Ok(ExitCode::from(if count == 0 { 1 } else { 0 })) +} + +/// Implements ripgrep's "generate" modes. +/// +/// These modes correspond to generating some kind of ancillary data related +/// to ripgrep. At present, this includes ripgrep's man page (in roff format) +/// and supported shell completions. +fn generate(mode: crate::flags::GenerateMode) -> anyhow::Result { + use crate::flags::GenerateMode; + + let output = match mode { + GenerateMode::Man => flags::generate_man_page(), + GenerateMode::CompleteBash => flags::generate_complete_bash(), + GenerateMode::CompleteZsh => flags::generate_complete_zsh(), + GenerateMode::CompleteFish => flags::generate_complete_fish(), + GenerateMode::CompletePowerShell => { + flags::generate_complete_powershell() + } + }; + writeln!(std::io::stdout(), "{}", output.trim_end())?; + Ok(ExitCode::from(0)) +} + +/// Implements ripgrep's "special" modes. +/// +/// A special mode is one that generally short-circuits most (not all) of +/// ripgrep's initialization logic and skips right to this routine. The +/// special modes essentially consist of printing help and version output. The +/// idea behind the short circuiting is to ensure there is as little as possible +/// (within reason) that would prevent ripgrep from emitting help output. +/// +/// For example, part of the initialization logic that is skipped (among +/// other things) is accessing the current working directory. If that fails, +/// ripgrep emits an error. We don't want to emit an error if it fails and +/// the user requested version or help information. +fn special(mode: crate::flags::SpecialMode) -> anyhow::Result { + use crate::flags::SpecialMode; + + let mut exit = ExitCode::from(0); + let output = match mode { + SpecialMode::HelpShort => flags::generate_help_short(), + SpecialMode::HelpLong => flags::generate_help_long(), + SpecialMode::VersionShort => flags::generate_version_short(), + SpecialMode::VersionLong => flags::generate_version_long(), + // --pcre2-version is a little special because it emits an error + // exit code if this build of ripgrep doesn't support PCRE2. + SpecialMode::VersionPCRE2 => { + let (output, available) = flags::generate_version_pcre2(); + if !available { + exit = ExitCode::from(1); + } + output + } + }; + writeln!(std::io::stdout(), "{}", output.trim_end())?; + Ok(exit) +} + +/// Prints a heuristic error messages when nothing is searched. +/// +/// This can happen if an applicable ignore file has one or more rules that +/// are too broad and cause ripgrep to ignore everything. +/// +/// We only show this error message when the user does *not* provide an +/// explicit path to search. This is because the message can otherwise be +/// noisy, e.g., when it is intended that there is nothing to search. +fn eprint_nothing_searched() { + err_message!( + "No files were searched, which means ripgrep probably \ + applied a filter you didn't expect.\n\ + Running with --debug will show why files are being skipped." + ); +} + +/// Prints the statistics given to the writer given. +/// +/// The search mode given determines whether the stats should be printed in +/// a plain text format or in a JSON format. +/// +/// The `started` time should be the time at which ripgrep started working. +/// +/// If an error occurs while writing, then writing stops and the error is +/// returned. Note that callers should probably ignore this error, since +/// whether stats fail to print or not generally shouldn't cause ripgrep to +/// enter into an "error" state. And usually the only way for this to fail is +/// if writing to stdout itself fails. +fn print_stats( + mode: SearchMode, + stats: &grep::printer::Stats, + started: std::time::Instant, + mut wtr: W, +) -> std::io::Result<()> { + let elapsed = std::time::Instant::now().duration_since(started); + if matches!(mode, SearchMode::JSON) { + // We specifically match the format laid out by the JSON printer in + // the grep-printer crate. We simply "extend" it with the 'summary' + // message type. + serde_json::to_writer( + &mut wtr, + &serde_json::json!({ + "type": "summary", + "data": { + "stats": stats, + "elapsed_total": { + "secs": elapsed.as_secs(), + "nanos": elapsed.subsec_nanos(), + "human": format!("{:0.6}s", elapsed.as_secs_f64()), + }, + } + }), + )?; + write!(wtr, "\n") + } else { + write!( + wtr, + " +{matches} matches +{lines} matched lines +{searches_with_match} files contained matches +{searches} files searched +{bytes_printed} bytes printed +{bytes_searched} bytes searched +{search_time:0.6} seconds spent searching +{process_time:0.6} seconds total +", + matches = stats.matches(), + lines = stats.matched_lines(), + searches_with_match = stats.searches_with_match(), + searches = stats.searches(), + bytes_printed = stats.bytes_printed(), + bytes_searched = stats.bytes_searched(), + search_time = stats.elapsed().as_secs_f64(), + process_time = elapsed.as_secs_f64(), + ) + } +} diff --git a/userland/ripgrep/crates/core/messages.rs b/userland/ripgrep/crates/core/messages.rs new file mode 100644 index 0000000..333e707 --- /dev/null +++ b/userland/ripgrep/crates/core/messages.rs @@ -0,0 +1,139 @@ +/*! +This module defines some macros and some light shared mutable state. + +This state is responsible for keeping track of whether we should emit certain +kinds of messages to the user (such as errors) that are distinct from the +standard "debug" or "trace" log messages. This state is specifically set at +startup time when CLI arguments are parsed and then never changed. + +The other state tracked here is whether ripgrep experienced an error +condition. Aside from errors associated with invalid CLI arguments, ripgrep +generally does not abort when an error occurs (e.g., if reading a file failed). +But when an error does occur, it will alter ripgrep's exit status. Thus, when +an error message is emitted via `err_message`, then a global flag is toggled +indicating that at least one error occurred. When ripgrep exits, this flag is +consulted to determine what the exit status ought to be. +*/ + +use std::sync::atomic::{AtomicBool, Ordering}; + +/// When false, "messages" will not be printed. +static MESSAGES: AtomicBool = AtomicBool::new(false); +/// When false, "messages" related to ignore rules will not be printed. +static IGNORE_MESSAGES: AtomicBool = AtomicBool::new(false); +/// Flipped to true when an error message is printed. +static ERRORED: AtomicBool = AtomicBool::new(false); + +/// Like eprintln, but locks stdout to prevent interleaving lines. +/// +/// This locks stdout, not stderr, even though this prints to stderr. This +/// avoids the appearance of interleaving output when stdout and stderr both +/// correspond to a tty. +#[macro_export] +macro_rules! eprintln_locked { + ($($tt:tt)*) => {{ + { + use std::io::Write; + + // This is a bit of an abstraction violation because we explicitly + // lock stdout before printing to stderr. This avoids interleaving + // lines within ripgrep because `search_parallel` uses `termcolor`, + // which accesses the same stdout lock when writing lines. + let stdout = std::io::stdout().lock(); + let mut stderr = std::io::stderr().lock(); + // We specifically ignore any errors here. One plausible error we + // can get in some cases is a broken pipe error. And when that + // occurs, we should exit gracefully. Otherwise, just abort with + // an error code because there isn't much else we can do. + // + // See: https://github.com/BurntSushi/ripgrep/issues/1966 + if let Err(err) = write!(stderr, "rg: ") { + if err.kind() == std::io::ErrorKind::BrokenPipe { + std::process::exit(0); + } else { + std::process::exit(2); + } + } + if let Err(err) = writeln!(stderr, $($tt)*) { + if err.kind() == std::io::ErrorKind::BrokenPipe { + std::process::exit(0); + } else { + std::process::exit(2); + } + } + drop(stdout); + } + }} +} + +/// Emit a non-fatal error message, unless messages were disabled. +#[macro_export] +macro_rules! message { + ($($tt:tt)*) => { + if crate::messages::messages() { + eprintln_locked!($($tt)*); + } + } +} + +/// Like message, but sets ripgrep's "errored" flag, which controls the exit +/// status. +#[macro_export] +macro_rules! err_message { + ($($tt:tt)*) => { + crate::messages::set_errored(); + message!($($tt)*); + } +} + +/// Emit a non-fatal ignore-related error message (like a parse error), unless +/// ignore-messages were disabled. +#[macro_export] +macro_rules! ignore_message { + ($($tt:tt)*) => { + if crate::messages::messages() && crate::messages::ignore_messages() { + eprintln_locked!($($tt)*); + } + } +} + +/// Returns true if and only if messages should be shown. +pub(crate) fn messages() -> bool { + MESSAGES.load(Ordering::Relaxed) +} + +/// Set whether messages should be shown or not. +/// +/// By default, they are not shown. +pub(crate) fn set_messages(yes: bool) { + MESSAGES.store(yes, Ordering::Relaxed) +} + +/// Returns true if and only if "ignore" related messages should be shown. +pub(crate) fn ignore_messages() -> bool { + IGNORE_MESSAGES.load(Ordering::Relaxed) +} + +/// Set whether "ignore" related messages should be shown or not. +/// +/// By default, they are not shown. +/// +/// Note that this is overridden if `messages` is disabled. Namely, if +/// `messages` is disabled, then "ignore" messages are never shown, regardless +/// of this setting. +pub(crate) fn set_ignore_messages(yes: bool) { + IGNORE_MESSAGES.store(yes, Ordering::Relaxed) +} + +/// Returns true if and only if ripgrep came across a non-fatal error. +pub(crate) fn errored() -> bool { + ERRORED.load(Ordering::Relaxed) +} + +/// Indicate that ripgrep has come across a non-fatal error. +/// +/// Callers should not use this directly. Instead, it is called automatically +/// via the `err_message` macro. +pub(crate) fn set_errored() { + ERRORED.store(true, Ordering::Relaxed); +} diff --git a/userland/ripgrep/crates/core/search.rs b/userland/ripgrep/crates/core/search.rs new file mode 100644 index 0000000..019ef09 --- /dev/null +++ b/userland/ripgrep/crates/core/search.rs @@ -0,0 +1,449 @@ +/*! +Defines a very high level "search worker" abstraction. + +A search worker manages the high level interaction points between the matcher +(i.e., which regex engine is used), the searcher (i.e., how data is actually +read and matched using the regex engine) and the printer. For example, the +search worker is where things like preprocessors or decompression happens. +*/ + +use std::{io, path::Path}; + +use {grep::matcher::Matcher, termcolor::WriteColor}; + +/// The configuration for the search worker. +/// +/// Among a few other things, the configuration primarily controls the way we +/// show search results to users at a very high level. +#[derive(Clone, Debug)] +struct Config { + preprocessor: Option, + preprocessor_globs: ignore::overrides::Override, + search_zip: bool, + binary_implicit: grep::searcher::BinaryDetection, + binary_explicit: grep::searcher::BinaryDetection, +} + +impl Default for Config { + fn default() -> Config { + Config { + preprocessor: None, + preprocessor_globs: ignore::overrides::Override::empty(), + search_zip: false, + binary_implicit: grep::searcher::BinaryDetection::none(), + binary_explicit: grep::searcher::BinaryDetection::none(), + } + } +} + +/// A builder for configuring and constructing a search worker. +#[derive(Clone, Debug)] +pub(crate) struct SearchWorkerBuilder { + config: Config, + command_builder: grep::cli::CommandReaderBuilder, +} + +impl Default for SearchWorkerBuilder { + fn default() -> SearchWorkerBuilder { + SearchWorkerBuilder::new() + } +} + +impl SearchWorkerBuilder { + /// Create a new builder for configuring and constructing a search worker. + pub(crate) fn new() -> SearchWorkerBuilder { + let mut command_builder = grep::cli::CommandReaderBuilder::new(); + command_builder.async_stderr(true); + + SearchWorkerBuilder { config: Config::default(), command_builder } + } + + /// Create a new search worker using the given searcher, matcher and + /// printer. + pub(crate) fn build( + &self, + matcher: PatternMatcher, + searcher: grep::searcher::Searcher, + printer: Printer, + ) -> SearchWorker { + let config = self.config.clone(); + let command_builder = self.command_builder.clone(); + let decomp_builder = config.search_zip.then(|| { + let mut decomp_builder = + grep::cli::DecompressionReaderBuilder::new(); + decomp_builder.async_stderr(true); + decomp_builder + }); + SearchWorker { + config, + command_builder, + decomp_builder, + matcher, + searcher, + printer, + } + } + + /// Set the path to a preprocessor command. + /// + /// When this is set, instead of searching files directly, the given + /// command will be run with the file path as the first argument, and the + /// output of that command will be searched instead. + pub(crate) fn preprocessor( + &mut self, + cmd: Option, + ) -> anyhow::Result<&mut SearchWorkerBuilder> { + if let Some(ref prog) = cmd { + let bin = grep::cli::resolve_binary(prog)?; + self.config.preprocessor = Some(bin); + } else { + self.config.preprocessor = None; + } + Ok(self) + } + + /// Set the globs for determining which files should be run through the + /// preprocessor. By default, with no globs and a preprocessor specified, + /// every file is run through the preprocessor. + pub(crate) fn preprocessor_globs( + &mut self, + globs: ignore::overrides::Override, + ) -> &mut SearchWorkerBuilder { + self.config.preprocessor_globs = globs; + self + } + + /// Enable the decompression and searching of common compressed files. + /// + /// When enabled, if a particular file path is recognized as a compressed + /// file, then it is decompressed before searching. + /// + /// Note that if a preprocessor command is set, then it overrides this + /// setting. + pub(crate) fn search_zip( + &mut self, + yes: bool, + ) -> &mut SearchWorkerBuilder { + self.config.search_zip = yes; + self + } + + /// Set the binary detection that should be used when searching files + /// found via a recursive directory search. + /// + /// Generally, this binary detection may be + /// `grep::searcher::BinaryDetection::quit` if we want to skip binary files + /// completely. + /// + /// By default, no binary detection is performed. + pub(crate) fn binary_detection_implicit( + &mut self, + detection: grep::searcher::BinaryDetection, + ) -> &mut SearchWorkerBuilder { + self.config.binary_implicit = detection; + self + } + + /// Set the binary detection that should be used when searching files + /// explicitly supplied by an end user. + /// + /// Generally, this binary detection should NOT be + /// `grep::searcher::BinaryDetection::quit`, since we never want to + /// automatically filter files supplied by the end user. + /// + /// By default, no binary detection is performed. + pub(crate) fn binary_detection_explicit( + &mut self, + detection: grep::searcher::BinaryDetection, + ) -> &mut SearchWorkerBuilder { + self.config.binary_explicit = detection; + self + } +} + +/// The result of executing a search. +/// +/// Generally speaking, the "result" of a search is sent to a printer, which +/// writes results to an underlying writer such as stdout or a file. However, +/// every search also has some aggregate statistics or meta data that may be +/// useful to higher level routines. +#[derive(Clone, Debug, Default)] +pub(crate) struct SearchResult { + has_match: bool, + stats: Option, +} + +impl SearchResult { + /// Whether the search found a match or not. + pub(crate) fn has_match(&self) -> bool { + self.has_match + } + + /// Return aggregate search statistics for a single search, if available. + /// + /// It can be expensive to compute statistics, so these are only present + /// if explicitly enabled in the printer provided by the caller. + pub(crate) fn stats(&self) -> Option<&grep::printer::Stats> { + self.stats.as_ref() + } +} + +/// The pattern matcher used by a search worker. +#[derive(Clone, Debug)] +pub(crate) enum PatternMatcher { + RustRegex(grep::regex::RegexMatcher), + #[cfg(feature = "pcre2")] + PCRE2(grep::pcre2::RegexMatcher), +} + +/// The printer used by a search worker. +/// +/// The `W` type parameter refers to the type of the underlying writer. +#[derive(Clone, Debug)] +pub(crate) enum Printer { + /// Use the standard printer, which supports the classic grep-like format. + Standard(grep::printer::Standard), + /// Use the summary printer, which supports aggregate displays of search + /// results. + Summary(grep::printer::Summary), + /// A JSON printer, which emits results in the JSON Lines format. + JSON(grep::printer::JSON), +} + +impl Printer { + /// Return a mutable reference to the underlying printer's writer. + pub(crate) fn get_mut(&mut self) -> &mut W { + match *self { + Printer::Standard(ref mut p) => p.get_mut(), + Printer::Summary(ref mut p) => p.get_mut(), + Printer::JSON(ref mut p) => p.get_mut(), + } + } +} + +/// A worker for executing searches. +/// +/// It is intended for a single worker to execute many searches, and is +/// generally intended to be used from a single thread. When searching using +/// multiple threads, it is better to create a new worker for each thread. +#[derive(Clone, Debug)] +pub(crate) struct SearchWorker { + config: Config, + command_builder: grep::cli::CommandReaderBuilder, + /// This is `None` when `search_zip` is not enabled, since in this case it + /// can never be used. We do this because building the reader can sometimes + /// do non-trivial work (like resolving the paths of decompression binaries + /// on Windows). + decomp_builder: Option, + matcher: PatternMatcher, + searcher: grep::searcher::Searcher, + printer: Printer, +} + +impl SearchWorker { + /// Execute a search over the given haystack. + pub(crate) fn search( + &mut self, + haystack: &crate::haystack::Haystack, + ) -> io::Result { + let bin = if haystack.is_explicit() { + self.config.binary_explicit.clone() + } else { + self.config.binary_implicit.clone() + }; + let path = haystack.path(); + log::trace!("{}: binary detection: {:?}", path.display(), bin); + + self.searcher.set_binary_detection(bin); + if haystack.is_stdin() { + self.search_reader(path, &mut io::stdin().lock()) + } else if self.should_preprocess(path) { + self.search_preprocessor(path) + } else if self.should_decompress(path) { + self.search_decompress(path) + } else { + self.search_path(path) + } + } + + /// Return a mutable reference to the underlying printer. + pub(crate) fn printer(&mut self) -> &mut Printer { + &mut self.printer + } + + /// Returns true if and only if the given file path should be + /// decompressed before searching. + fn should_decompress(&self, path: &Path) -> bool { + self.decomp_builder.as_ref().is_some_and(|decomp_builder| { + decomp_builder.get_matcher().has_command(path) + }) + } + + /// Returns true if and only if the given file path should be run through + /// the preprocessor. + fn should_preprocess(&self, path: &Path) -> bool { + if !self.config.preprocessor.is_some() { + return false; + } + if self.config.preprocessor_globs.is_empty() { + return true; + } + !self.config.preprocessor_globs.matched(path, false).is_ignore() + } + + /// Search the given file path by first asking the preprocessor for the + /// data to search instead of opening the path directly. + fn search_preprocessor( + &mut self, + path: &Path, + ) -> io::Result { + use std::{fs::File, process::Stdio}; + + let bin = self.config.preprocessor.as_ref().unwrap(); + let mut cmd = std::process::Command::new(bin); + cmd.arg(path).stdin(Stdio::from(File::open(path)?)); + + let mut rdr = self.command_builder.build(&mut cmd).map_err(|err| { + io::Error::new( + io::ErrorKind::Other, + format!( + "preprocessor command could not start: '{cmd:?}': {err}", + ), + ) + })?; + let result = self.search_reader(path, &mut rdr).map_err(|err| { + io::Error::new( + io::ErrorKind::Other, + format!("preprocessor command failed: '{cmd:?}': {err}"), + ) + }); + let close_result = rdr.close(); + let search_result = result?; + close_result?; + Ok(search_result) + } + + /// Attempt to decompress the data at the given file path and search the + /// result. If the given file path isn't recognized as a compressed file, + /// then search it without doing any decompression. + fn search_decompress(&mut self, path: &Path) -> io::Result { + let Some(ref decomp_builder) = self.decomp_builder else { + return self.search_path(path); + }; + let mut rdr = decomp_builder.build(path)?; + let result = self.search_reader(path, &mut rdr); + let close_result = rdr.close(); + let search_result = result?; + close_result?; + Ok(search_result) + } + + /// Search the contents of the given file path. + fn search_path(&mut self, path: &Path) -> io::Result { + use self::PatternMatcher::*; + + let (searcher, printer) = (&mut self.searcher, &mut self.printer); + match self.matcher { + RustRegex(ref m) => search_path(m, searcher, printer, path), + #[cfg(feature = "pcre2")] + PCRE2(ref m) => search_path(m, searcher, printer, path), + } + } + + /// Executes a search on the given reader, which may or may not correspond + /// directly to the contents of the given file path. Instead, the reader + /// may actually cause something else to be searched (for example, when + /// a preprocessor is set or when decompression is enabled). In those + /// cases, the file path is used for visual purposes only. + /// + /// Generally speaking, this method should only be used when there is no + /// other choice. Searching via `search_path` provides more opportunities + /// for optimizations (such as memory maps). + fn search_reader( + &mut self, + path: &Path, + rdr: &mut R, + ) -> io::Result { + use self::PatternMatcher::*; + + let (searcher, printer) = (&mut self.searcher, &mut self.printer); + match self.matcher { + RustRegex(ref m) => search_reader(m, searcher, printer, path, rdr), + #[cfg(feature = "pcre2")] + PCRE2(ref m) => search_reader(m, searcher, printer, path, rdr), + } + } +} + +/// Search the contents of the given file path using the given matcher, +/// searcher and printer. +fn search_path( + matcher: M, + searcher: &mut grep::searcher::Searcher, + printer: &mut Printer, + path: &Path, +) -> io::Result { + match *printer { + Printer::Standard(ref mut p) => { + let mut sink = p.sink_with_path(&matcher, path); + searcher.search_path(&matcher, path, &mut sink)?; + Ok(SearchResult { + has_match: sink.has_match(), + stats: sink.stats().cloned(), + }) + } + Printer::Summary(ref mut p) => { + let mut sink = p.sink_with_path(&matcher, path); + searcher.search_path(&matcher, path, &mut sink)?; + Ok(SearchResult { + has_match: sink.has_match(), + stats: sink.stats().cloned(), + }) + } + Printer::JSON(ref mut p) => { + let mut sink = p.sink_with_path(&matcher, path); + searcher.search_path(&matcher, path, &mut sink)?; + Ok(SearchResult { + has_match: sink.has_match(), + stats: Some(sink.stats().clone()), + }) + } + } +} + +/// Search the contents of the given reader using the given matcher, searcher +/// and printer. +fn search_reader( + matcher: M, + searcher: &mut grep::searcher::Searcher, + printer: &mut Printer, + path: &Path, + mut rdr: R, +) -> io::Result { + match *printer { + Printer::Standard(ref mut p) => { + let mut sink = p.sink_with_path(&matcher, path); + searcher.search_reader(&matcher, &mut rdr, &mut sink)?; + Ok(SearchResult { + has_match: sink.has_match(), + stats: sink.stats().cloned(), + }) + } + Printer::Summary(ref mut p) => { + let mut sink = p.sink_with_path(&matcher, path); + searcher.search_reader(&matcher, &mut rdr, &mut sink)?; + Ok(SearchResult { + has_match: sink.has_match(), + stats: sink.stats().cloned(), + }) + } + Printer::JSON(ref mut p) => { + let mut sink = p.sink_with_path(&matcher, path); + searcher.search_reader(&matcher, &mut rdr, &mut sink)?; + Ok(SearchResult { + has_match: sink.has_match(), + stats: Some(sink.stats().clone()), + }) + } + } +} diff --git a/userland/ripgrep/pkg/windows/Manifest.xml b/userland/ripgrep/pkg/windows/Manifest.xml new file mode 100644 index 0000000..b6f0e70 --- /dev/null +++ b/userland/ripgrep/pkg/windows/Manifest.xml @@ -0,0 +1,28 @@ + + + + + + + + + + + + + + + + UTF-8 + + + + + + true + + + diff --git a/userland/ripgrep/pkg/windows/README.md b/userland/ripgrep/pkg/windows/README.md new file mode 100644 index 0000000..7be701b --- /dev/null +++ b/userland/ripgrep/pkg/windows/README.md @@ -0,0 +1,15 @@ +This directory contains a Windows manifest for various Windows-specific +settings. + +The main thing we enable here is [`longPathAware`], which permits paths of the +form `C:\` to be longer than 260 characters. + +The approach taken here was modeled off of a [similar change for `rustc`][rustc pr]. +In particular, this manifest gets linked into the final binary. Those linker +arguments are applied in `build.rs`. + +This currently only applies to MSVC builds. If there's an easy way to make this +apply to GNU builds as well, then patches are welcome. + +[`longPathAware`]: https://learn.microsoft.com/en-us/windows/win32/sbscs/application-manifests#longpathaware +[rustc pr]: https://github.com/rust-lang/rust/pull/96737 diff --git a/userland/ripgrep/rustfmt.toml b/userland/ripgrep/rustfmt.toml new file mode 100644 index 0000000..bfbf0b4 --- /dev/null +++ b/userland/ripgrep/rustfmt.toml @@ -0,0 +1,3 @@ +max_width = 79 +use_small_heuristics = "max" +edition = "2024" diff --git a/userland/ripgrep/tests/binary.rs b/userland/ripgrep/tests/binary.rs new file mode 100644 index 0000000..73639c1 --- /dev/null +++ b/userland/ripgrep/tests/binary.rs @@ -0,0 +1,468 @@ +use crate::util::{Dir, TestCommand}; + +// This file contains a smattering of tests specifically for checking ripgrep's +// handling of binary files. There's quite a bit of discussion on this in this +// bug report: https://github.com/BurntSushi/ripgrep/issues/306 + +// Our haystack is the first 2,133 lines of Gutenberg's copy of "A Study in +// Scarlet," with a NUL byte at line 1870: `abcdef\x00`. +// +// The position and size of the haystack is, unfortunately, significant. In +// particular, the NUL byte is specifically inserted at some point *after* the +// first 65,536 bytes, which corresponds to the initial capacity of the buffer +// that ripgrep uses to read files. (grep for DEFAULT_BUFFER_CAPACITY.) The +// position of the NUL byte ensures that we can execute some search on the +// initial buffer contents without ever detecting any binary data. Moreover, +// when using a memory map for searching, only the first 65,536 bytes are +// scanned for a NUL byte, so no binary bytes are detected at all when using +// a memory map (unless our query matches line 1898). +// +// One last note: in the tests below, we use --no-mmap heavily because binary +// detection with memory maps is a bit different. Namely, NUL bytes are only +// searched for in the first few KB of the file and in a match. Normally, NUL +// bytes are searched for everywhere. +const HAY: &'static [u8] = include_bytes!("./data/sherlock-nul.txt"); + +// Tests for binary file detection when using memory maps. +// As noted in the original comments, with memory maps binary detection +// works differently - NUL bytes are only searched for in the first few KB +// of the file and in matches. +// +// Note that we don't run these on macOS, which has memory maps forcefully +// disabled because they suck so much. + +// Test that matches in a binary file with memory maps work as expected +// with implicit file search (via glob pattern). +#[cfg(not(target_os = "macos"))] +rgtest!(mmap_match_implicit, |dir: Dir, mut cmd: TestCommand| { + dir.create_bytes("hay", HAY); + cmd.args(&["--mmap", "-n", "Project Gutenberg EBook", "-g", "hay"]); + + // With mmap, we get a match and a warning about binary content + let expected = "\ +hay:1:The Project Gutenberg EBook of A Study In Scarlet, by Arthur Conan Doyle +"; + eqnice!(expected, cmd.stdout()); +}); + +// Test with an explicit file argument when using memory maps. +#[cfg(not(target_os = "macos"))] +rgtest!(mmap_match_explicit, |dir: Dir, mut cmd: TestCommand| { + dir.create_bytes("hay", HAY); + cmd.args(&["--mmap", "-n", "Project Gutenberg EBook", "hay"]); + + let expected = "\ +1:The Project Gutenberg EBook of A Study In Scarlet, by Arthur Conan Doyle +"; + eqnice!(expected, cmd.stdout()); +}); + +// Test specifically with a pattern that matches near the NUL byte which should +// trigger binary detection with memory maps. +#[cfg(not(target_os = "macos"))] +rgtest!(mmap_match_near_nul, |dir: Dir, mut cmd: TestCommand| { + dir.create_bytes("hay", HAY); + // Pattern that matches around line 1898 where the NUL byte is. + // Note: Using direct file path instead of glob. + cmd.args(&["--mmap", "-n", "abcdef", "hay"]); + + let expected = "\ +binary file matches (found \"\\0\" byte around offset 77041) +"; + eqnice!(expected, cmd.stdout()); +}); + +// Test with --count option to ensure full file scanning works with mmap. +#[cfg(not(target_os = "macos"))] +rgtest!(mmap_match_count, |dir: Dir, mut cmd: TestCommand| { + dir.create_bytes("hay", HAY); + cmd.args(&["--mmap", "-c", "Project Gutenberg EBook|Heaven", "hay"]); + + // With mmap, since we're counting all matches and might not + // encounter the NUL byte during initial detection, the count + // should still be reported. + eqnice!("2\n", cmd.stdout()); +}); + +// Test binary detection with mmap when pattern would match before and after NUL +// byte. +#[cfg(not(target_os = "macos"))] +rgtest!(mmap_match_multiple, |dir: Dir, mut cmd: TestCommand| { + dir.create_bytes("hay", HAY); + // Use explicit file path. + cmd.args(&["--mmap", "-n", "Project Gutenberg EBook|Heaven", "hay"]); + + // With explicit file and memory maps, matches before and after NUL byte + // are shown. + let expected = "\ +1:The Project Gutenberg EBook of A Study In Scarlet, by Arthur Conan Doyle +1871:\"No. Heaven knows what the objects of his studies are. But here we +"; + eqnice!(expected, cmd.stdout()); +}); + +// Test that --binary flag can have odd results when searching with a memory +// map. +#[cfg(not(target_os = "macos"))] +rgtest!(mmap_binary_flag, |dir: Dir, mut cmd: TestCommand| { + dir.create_bytes("hay", HAY); + // Use glob pattern. + cmd.args(&["--mmap", "-n", "--binary", "Heaven", "-g", "hay"]); + + let expected = "\ +hay:1871:\"No. Heaven knows what the objects of his studies are. But here we +"; + eqnice!(expected, cmd.stdout()); +}); + +// Test that using -a/--text flag works as expected with mmap. +#[cfg(not(target_os = "macos"))] +rgtest!(mmap_text_flag, |dir: Dir, mut cmd: TestCommand| { + dir.create_bytes("hay", HAY); + cmd.args(&["--mmap", "-n", "--text", "Heaven", "-g", "hay"]); + + // With --text flag, binary detection should be disabled. + let expected = "\ +hay:1871:\"No. Heaven knows what the objects of his studies are. But here we +"; + eqnice!(expected, cmd.stdout()); +}); + +// Test pattern that matches before and after the NUL byte with memory maps. +#[cfg(not(target_os = "macos"))] +rgtest!(mmap_after_nul_match, |dir: Dir, mut cmd: TestCommand| { + dir.create_bytes("hay", HAY); + // Use explicit file path. + cmd.args(&["--mmap", "-n", "medical student", "hay"]); + + // With explicit file and memory maps, all matches are shown + let expected = "\ +176:\"A medical student, I suppose?\" said I. +409:\"A medical student, I suppose?\" said I. +642:\"A medical student, I suppose?\" said I. +875:\"A medical student, I suppose?\" said I. +1108:\"A medical student, I suppose?\" said I. +1341:\"A medical student, I suppose?\" said I. +1574:\"A medical student, I suppose?\" said I. +1807:\"A medical student, I suppose?\" said I. +1867:\"And yet you say he is not a medical student?\" +"; + eqnice!(expected, cmd.stdout()); +}); + +// This tests that ripgrep prints a warning message if it finds and prints a +// match in a binary file before detecting that it is a binary file. The point +// here is to notify that user that the search of the file is only partially +// complete. +// +// This applies to files that are *implicitly* searched via a recursive +// directory traversal. In particular, this results in a WARNING message being +// printed. We make our file "implicit" by doing a recursive search with a glob +// that matches our file. +rgtest!(after_match1_implicit, |dir: Dir, mut cmd: TestCommand| { + dir.create_bytes("hay", HAY); + cmd.args(&["--no-mmap", "-n", "Project Gutenberg EBook", "-g", "hay"]); + + let expected = "\ +hay:1:The Project Gutenberg EBook of A Study In Scarlet, by Arthur Conan Doyle +hay: WARNING: stopped searching binary file after match (found \"\\0\" byte around offset 77041) +"; + eqnice!(expected, cmd.stdout()); +}); + +// Like after_match1_implicit, except we provide a file to search +// explicitly. This results in identical behavior, but a different message. +rgtest!(after_match1_explicit, |dir: Dir, mut cmd: TestCommand| { + dir.create_bytes("hay", HAY); + cmd.args(&["--no-mmap", "-n", "Project Gutenberg EBook", "hay"]); + + let expected = "\ +1:The Project Gutenberg EBook of A Study In Scarlet, by Arthur Conan Doyle +binary file matches (found \"\\0\" byte around offset 77041) +"; + eqnice!(expected, cmd.stdout()); +}); + +// Like after_match1_explicit, except we feed our content on stdin. +rgtest!(after_match1_stdin, |_: Dir, mut cmd: TestCommand| { + cmd.args(&["--no-mmap", "-n", "Project Gutenberg EBook"]); + + let expected = "\ +1:The Project Gutenberg EBook of A Study In Scarlet, by Arthur Conan Doyle +binary file matches (found \"\\0\" byte around offset 77041) +"; + eqnice!(expected, cmd.pipe(HAY)); +}); + +// Like after_match1_implicit, but provides the --binary flag, which +// disables binary filtering. Thus, this matches the behavior of ripgrep as +// if the file were given explicitly. +rgtest!(after_match1_implicit_binary, |dir: Dir, mut cmd: TestCommand| { + dir.create_bytes("hay", HAY); + cmd.args(&[ + "--no-mmap", + "-n", + "--binary", + "Project Gutenberg EBook", + "-g", + "hay", + ]); + + let expected = "\ +hay:1:The Project Gutenberg EBook of A Study In Scarlet, by Arthur Conan Doyle +hay: binary file matches (found \"\\0\" byte around offset 77041) +"; + eqnice!(expected, cmd.stdout()); +}); + +// Like after_match1_implicit, but enables -a/--text, so no binary +// detection should be performed. +rgtest!(after_match1_implicit_text, |dir: Dir, mut cmd: TestCommand| { + dir.create_bytes("hay", HAY); + cmd.args(&[ + "--no-mmap", + "-n", + "--text", + "Project Gutenberg EBook", + "-g", + "hay", + ]); + + let expected = "\ +hay:1:The Project Gutenberg EBook of A Study In Scarlet, by Arthur Conan Doyle +"; + eqnice!(expected, cmd.stdout()); +}); + +// Like after_match1_implicit_text, but enables -a/--text, so no binary +// detection should be performed. +rgtest!(after_match1_explicit_text, |dir: Dir, mut cmd: TestCommand| { + dir.create_bytes("hay", HAY); + cmd.args(&["--no-mmap", "-n", "--text", "Project Gutenberg EBook", "hay"]); + + let expected = "\ +1:The Project Gutenberg EBook of A Study In Scarlet, by Arthur Conan Doyle +"; + eqnice!(expected, cmd.stdout()); +}); + +// Like after_match1_implicit, except this asks ripgrep to print all matching +// files. +// +// This is an interesting corner case that one might consider a bug, however, +// it's unlikely to be fixed. Namely, ripgrep probably shouldn't print `hay` +// as a matching file since it is in fact a binary file, and thus should be +// filtered out by default. However, the --files-with-matches flag will print +// out the path of a matching file as soon as a match is seen and then stop +// searching completely. Therefore, the NUL byte is never actually detected. +// +// The only way to fix this would be to kill ripgrep's performance in this case +// and continue searching the entire file for a NUL byte. (Similarly if the +// --quiet flag is set. See the next test.) +rgtest!(after_match1_implicit_path, |dir: Dir, mut cmd: TestCommand| { + dir.create_bytes("hay", HAY); + cmd.args(&["--no-mmap", "-l", "Project Gutenberg EBook", "-g", "hay"]); + eqnice!("hay\n", cmd.stdout()); +}); + +// Like after_match1_implicit_path, except this indicates that a match was +// found with no other output. (This is the same bug described above, but +// manifest as an exit code with no output.) +rgtest!(after_match1_implicit_quiet, |dir: Dir, mut cmd: TestCommand| { + dir.create_bytes("hay", HAY); + cmd.args(&["--no-mmap", "-q", "Project Gutenberg EBook", "-g", "hay"]); + eqnice!("", cmd.stdout()); +}); + +// This sets up the same test as after_match1_implicit_path, but instead of +// just printing the matching files, this includes the full count of matches. +// In this case, we need to search the entire file, so ripgrep correctly +// detects the binary data and suppresses output. +rgtest!(after_match1_implicit_count, |dir: Dir, mut cmd: TestCommand| { + dir.create_bytes("hay", HAY); + cmd.args(&["--no-mmap", "-c", "Project Gutenberg EBook", "-g", "hay"]); + cmd.assert_err(); +}); + +// Like after_match1_implicit_count, except the --binary flag is provided, +// which makes ripgrep disable binary data filtering even for implicit files. +rgtest!( + after_match1_implicit_count_binary, + |dir: Dir, mut cmd: TestCommand| { + dir.create_bytes("hay", HAY); + cmd.args(&[ + "--no-mmap", + "-c", + "--binary", + "Project Gutenberg EBook", + "-g", + "hay", + ]); + eqnice!("hay:1\n", cmd.stdout()); + } +); + +// Like after_match1_implicit_count, except the file path is provided +// explicitly, so binary filtering is disabled and a count is correctly +// reported. +rgtest!(after_match1_explicit_count, |dir: Dir, mut cmd: TestCommand| { + dir.create_bytes("hay", HAY); + cmd.args(&["--no-mmap", "-c", "Project Gutenberg EBook", "hay"]); + eqnice!("1\n", cmd.stdout()); +}); + +// This tests that a match way before the NUL byte is shown, but a match after +// the NUL byte is not. +rgtest!(after_match2_implicit, |dir: Dir, mut cmd: TestCommand| { + dir.create_bytes("hay", HAY); + cmd.args(&[ + "--no-mmap", + "-n", + "Project Gutenberg EBook|a medical student", + "-g", + "hay", + ]); + + let expected = "\ +hay:1:The Project Gutenberg EBook of A Study In Scarlet, by Arthur Conan Doyle +hay: WARNING: stopped searching binary file after match (found \"\\0\" byte around offset 77041) +"; + eqnice!(expected, cmd.stdout()); +}); + +// Like after_match2_implicit, but enables -a/--text, so no binary +// detection should be performed. +rgtest!(after_match2_implicit_text, |dir: Dir, mut cmd: TestCommand| { + dir.create_bytes("hay", HAY); + cmd.args(&[ + "--no-mmap", + "-n", + "--text", + "Project Gutenberg EBook|a medical student", + "-g", + "hay", + ]); + + let expected = "\ +hay:1:The Project Gutenberg EBook of A Study In Scarlet, by Arthur Conan Doyle +hay:1867:\"And yet you say he is not a medical student?\" +"; + eqnice!(expected, cmd.stdout()); +}); + +// This tests that ripgrep *silently* quits before finding a match that occurs +// after a NUL byte. +rgtest!(before_match1_implicit, |dir: Dir, mut cmd: TestCommand| { + dir.create_bytes("hay", HAY); + cmd.args(&["--no-mmap", "-n", "Heaven", "-g", "hay"]); + cmd.assert_err(); +}); + +// This tests that ripgrep *does not* silently quit before finding a match that +// occurs after a NUL byte when a file is explicitly searched. +rgtest!(before_match1_explicit, |dir: Dir, mut cmd: TestCommand| { + dir.create_bytes("hay", HAY); + cmd.args(&["--no-mmap", "-n", "Heaven", "hay"]); + + let expected = "\ +binary file matches (found \"\\0\" byte around offset 77041) +"; + eqnice!(expected, cmd.stdout()); +}); + +// Like before_match1_implicit, but enables the --binary flag, which +// disables binary filtering. Thus, this matches the behavior of ripgrep as if +// the file were given explicitly. +rgtest!(before_match1_implicit_binary, |dir: Dir, mut cmd: TestCommand| { + dir.create_bytes("hay", HAY); + cmd.args(&["--no-mmap", "-n", "--binary", "Heaven", "-g", "hay"]); + + let expected = "\ +hay: binary file matches (found \"\\0\" byte around offset 77041) +"; + eqnice!(expected, cmd.stdout()); +}); + +// Like before_match1_implicit, but enables -a/--text, so no binary +// detection should be performed. +rgtest!(before_match1_implicit_text, |dir: Dir, mut cmd: TestCommand| { + dir.create_bytes("hay", HAY); + cmd.args(&["--no-mmap", "-n", "--text", "Heaven", "-g", "hay"]); + + let expected = "\ +hay:1871:\"No. Heaven knows what the objects of his studies are. But here we +"; + eqnice!(expected, cmd.stdout()); +}); + +// This tests that ripgrep *silently* quits before finding a match that occurs +// before a NUL byte, but within the same buffer as the NUL byte. +rgtest!(before_match2_implicit, |dir: Dir, mut cmd: TestCommand| { + dir.create_bytes("hay", HAY); + cmd.args(&["--no-mmap", "-n", "a medical student", "-g", "hay"]); + cmd.assert_err(); +}); + +// This tests that ripgrep *does not* silently quit before finding a match that +// occurs before a NUL byte, but within the same buffer as the NUL byte. Even +// though the match occurs before the NUL byte, ripgrep still doesn't print it +// because it has already scanned ahead to detect the NUL byte. (This matches +// the behavior of GNU grep.) +rgtest!(before_match2_explicit, |dir: Dir, mut cmd: TestCommand| { + dir.create_bytes("hay", HAY); + cmd.args(&["--no-mmap", "-n", "a medical student", "hay"]); + + let expected = "\ +binary file matches (found \"\\0\" byte around offset 77041) +"; + eqnice!(expected, cmd.stdout()); +}); + +// Like before_match1_implicit, but enables -a/--text, so no binary +// detection should be performed. +rgtest!(before_match2_implicit_text, |dir: Dir, mut cmd: TestCommand| { + dir.create_bytes("hay", HAY); + cmd.args(&["--no-mmap", "-n", "--text", "a medical student", "-g", "hay"]); + + let expected = "\ +hay:1867:\"And yet you say he is not a medical student?\" +"; + eqnice!(expected, cmd.stdout()); +}); + +// See: https://github.com/BurntSushi/ripgrep/issues/3131 +rgtest!( + matching_files_inconsistent_with_count, + |dir: Dir, _cmd: TestCommand| { + let mut file1 = String::new(); + file1.push_str("cat here\n"); + for _ in 0..150_000 { + file1.push_str("padding line\n"); + } + file1.push_str("\x00"); + + dir.create("file1.txt", &file1); + dir.create("file2.txt", "cat here"); + + let got = dir.command().args(&["--sort=path", "-l", "cat"]).stdout(); + eqnice!("file1.txt\nfile2.txt\n", got); + + // This is the inconsistent result that can't really be avoided without + // either making `-l/--files-with-matches` much slower or changing + // what "binary filtering" means. + let got = dir.command().args(&["--sort=path", "-c", "cat"]).stdout(); + eqnice!("file2.txt:1\n", got); + + let got = dir + .command() + .args(&["--sort=path", "-c", "cat", "--binary"]) + .stdout(); + eqnice!("file1.txt:1\nfile2.txt:1\n", got); + + let got = dir + .command() + .args(&["--sort=path", "-c", "cat", "--text"]) + .stdout(); + eqnice!("file1.txt:1\nfile2.txt:1\n", got); + } +); diff --git a/userland/ripgrep/tests/data/sherlock-nul.txt b/userland/ripgrep/tests/data/sherlock-nul.txt new file mode 100644 index 0000000..60fb2b0 --- /dev/null +++ b/userland/ripgrep/tests/data/sherlock-nul.txt @@ -0,0 +1,2133 @@ +The Project Gutenberg EBook of A Study In Scarlet, by Arthur Conan Doyle + +This eBook is for the use of anyone anywhere at no cost and with +almost no restrictions whatsoever. You may copy it, give it away or +re-use it under the terms of the Project Gutenberg License included +with this eBook or online at www.gutenberg.org + + +Title: A Study In Scarlet + +Author: Arthur Conan Doyle + +Posting Date: July 12, 2008 [EBook #244] +Release Date: April, 1995 +[Last updated: February 17, 2013] + +Language: English + + +*** START OF THIS PROJECT GUTENBERG EBOOK A STUDY IN SCARLET *** + + + + +Produced by Roger Squires + + + + + +A STUDY IN SCARLET. + +By A. Conan Doyle + +[1] + + + + Original Transcriber's Note: This etext is prepared directly + from an 1887 edition, and care has been taken to duplicate the + original exactly, including typographical and punctuation + vagaries. + + Additions to the text include adding the underscore character to + indicate italics, and textual end-notes in square braces. + + Project Gutenberg Editor's Note: In reproofing and moving old PG + files such as this to the present PG directory system it is the + policy to reformat the text to conform to present PG Standards. + In this case however, in consideration of the note above of the + original transcriber describing his care to try to duplicate the + original 1887 edition as to typography and punctuation vagaries, + no changes have been made in this ascii text file. However, in + the Latin-1 file and this html file, present standards are + followed and the several French and Spanish words have been + given their proper accents. + + Part II, The Country of the Saints, deals much with the Mormon Church. + + + + +A STUDY IN SCARLET. + + + + + +PART I. + +(_Being a reprint from the reminiscences of_ JOHN H. WATSON, M.D., _late +of the Army Medical Department._) [2] + + + + +CHAPTER I. MR. SHERLOCK HOLMES. + + +IN the year 1878 I took my degree of Doctor of Medicine of the +University of London, and proceeded to Netley to go through the course +prescribed for surgeons in the army. Having completed my studies there, +I was duly attached to the Fifth Northumberland Fusiliers as Assistant +Surgeon. The regiment was stationed in India at the time, and before +I could join it, the second Afghan war had broken out. On landing at +Bombay, I learned that my corps had advanced through the passes, and +was already deep in the enemy's country. I followed, however, with many +other officers who were in the same situation as myself, and succeeded +in reaching Candahar in safety, where I found my regiment, and at once +entered upon my new duties. + +The campaign brought honours and promotion to many, but for me it had +nothing but misfortune and disaster. I was removed from my brigade and +attached to the Berkshires, with whom I served at the fatal battle of +Maiwand. There I was struck on the shoulder by a Jezail bullet, which +shattered the bone and grazed the subclavian artery. I should have +fallen into the hands of the murderous Ghazis had it not been for the +devotion and courage shown by Murray, my orderly, who threw me across a +pack-horse, and succeeded in bringing me safely to the British lines. + +Worn with pain, and weak from the prolonged hardships which I had +undergone, I was removed, with a great train of wounded sufferers, to +the base hospital at Peshawar. Here I rallied, and had already improved +so far as to be able to walk about the wards, and even to bask a little +upon the verandah, when I was struck down by enteric fever, that curse +of our Indian possessions. For months my life was despaired of, and +when at last I came to myself and became convalescent, I was so weak and +emaciated that a medical board determined that not a day should be lost +in sending me back to England. I was dispatched, accordingly, in the +troopship "Orontes," and landed a month later on Portsmouth jetty, with +my health irretrievably ruined, but with permission from a paternal +government to spend the next nine months in attempting to improve it. + +I had neither kith nor kin in England, and was therefore as free as +air--or as free as an income of eleven shillings and sixpence a day will +permit a man to be. Under such circumstances, I naturally gravitated to +London, that great cesspool into which all the loungers and idlers of +the Empire are irresistibly drained. There I stayed for some time at +a private hotel in the Strand, leading a comfortless, meaningless +existence, and spending such money as I had, considerably more freely +than I ought. So alarming did the state of my finances become, that +I soon realized that I must either leave the metropolis and rusticate +somewhere in the country, or that I must make a complete alteration in +my style of living. Choosing the latter alternative, I began by making +up my mind to leave the hotel, and to take up my quarters in some less +pretentious and less expensive domicile. + +On the very day that I had come to this conclusion, I was standing at +the Criterion Bar, when some one tapped me on the shoulder, and turning +round I recognized young Stamford, who had been a dresser under me at +Barts. The sight of a friendly face in the great wilderness of London is +a pleasant thing indeed to a lonely man. In old days Stamford had never +been a particular crony of mine, but now I hailed him with enthusiasm, +and he, in his turn, appeared to be delighted to see me. In the +exuberance of my joy, I asked him to lunch with me at the Holborn, and +we started off together in a hansom. + +"Whatever have you been doing with yourself, Watson?" he asked in +undisguised wonder, as we rattled through the crowded London streets. +"You are as thin as a lath and as brown as a nut." + +I gave him a short sketch of my adventures, and had hardly concluded it +by the time that we reached our destination. + +"Poor devil!" he said, commiseratingly, after he had listened to my +misfortunes. "What are you up to now?" + +"Looking for lodgings." [3] I answered. "Trying to solve the problem +as to whether it is possible to get comfortable rooms at a reasonable +price." + +"That's a strange thing," remarked my companion; "you are the second man +to-day that has used that expression to me." + +"And who was the first?" I asked. + +"A fellow who is working at the chemical laboratory up at the hospital. +He was bemoaning himself this morning because he could not get someone +to go halves with him in some nice rooms which he had found, and which +were too much for his purse." + +"By Jove!" I cried, "if he really wants someone to share the rooms and +the expense, I am the very man for him. I should prefer having a partner +to being alone." + +Young Stamford looked rather strangely at me over his wine-glass. "You +don't know Sherlock Holmes yet," he said; "perhaps you would not care +for him as a constant companion." + +"Why, what is there against him?" + +"Oh, I didn't say there was anything against him. He is a little queer +in his ideas--an enthusiast in some branches of science. As far as I +know he is a decent fellow enough." + +"A medical student, I suppose?" said I. + +"No--I have no idea what he intends to go in for. I believe he is well +up in anatomy, and he is a first-class chemist; but, as far as I know, +he has never taken out any systematic medical classes. His studies are +very desultory and eccentric, but he has amassed a lot of out-of-the way +knowledge which would astonish his professors." + +"Did you never ask him what he was going in for?" I asked. + +"No; he is not a man that it is easy to draw out, though he can be +communicative enough when the fancy seizes him." + +"I should like to meet him," I said. "If I am to lodge with anyone, I +should prefer a man of studious and quiet habits. I am not strong +enough yet to stand much noise or excitement. I had enough of both in +Afghanistan to last me for the remainder of my natural existence. How +could I meet this friend of yours?" + +"He is sure to be at the laboratory," returned my companion. "He either +avoids the place for weeks, or else he works there from morning to +night. If you like, we shall drive round together after luncheon." + +"Certainly," I answered, and the conversation drifted away into other +channels. + +As we made our way to the hospital after leaving the Holborn, Stamford +gave me a few more particulars about the gentleman whom I proposed to +take as a fellow-lodger. + +"You mustn't blame me if you don't get on with him," he said; "I know +nothing more of him than I have learned from meeting him occasionally in +the laboratory. You proposed this arrangement, so you must not hold me +responsible." + +"If we don't get on it will be easy to part company," I answered. "It +seems to me, Stamford," I added, looking hard at my companion, "that you +have some reason for washing your hands of the matter. Is this fellow's +temper so formidable, or what is it? Don't be mealy-mouthed about it." + +"It is not easy to express the inexpressible," he answered with a laugh. +"Holmes is a little too scientific for my tastes--it approaches to +cold-bloodedness. I could imagine his giving a friend a little pinch of +the latest vegetable alkaloid, not out of malevolence, you understand, +but simply out of a spirit of inquiry in order to have an accurate idea +of the effects. To do him justice, I think that he would take it himself +with the same readiness. He appears to have a passion for definite and +exact knowledge." + +"Very right too." + +"Yes, but it may be pushed to excess. When it comes to beating the +subjects in the dissecting-rooms with a stick, it is certainly taking +rather a bizarre shape." + +"Beating the subjects!" + +"Yes, to verify how far bruises may be produced after death. I saw him +at it with my own eyes." + +This eBook is for the use of anyone anywhere at no cost and with +almost no restrictions whatsoever. You may copy it, give it away or +re-use it under the terms of the Project Gutenberg License included +with this eBook or online at www.gutenberg.org + + +Title: A Study In Scarlet + +Author: Arthur Conan Doyle + +Posting Date: July 12, 2008 [EBook #244] +Release Date: April, 1995 +[Last updated: February 17, 2013] + +Language: English + + +*** START OF THIS PROJECT GUTENBERG EBOOK A STUDY IN SCARLET *** + + + + +Produced by Roger Squires + + + + + +A STUDY IN SCARLET. + +By A. Conan Doyle + +[1] + + + + Original Transcriber's Note: This etext is prepared directly + from an 1887 edition, and care has been taken to duplicate the + original exactly, including typographical and punctuation + vagaries. + + Additions to the text include adding the underscore character to + indicate italics, and textual end-notes in square braces. + + Project Gutenberg Editor's Note: In reproofing and moving old PG + files such as this to the present PG directory system it is the + policy to reformat the text to conform to present PG Standards. + In this case however, in consideration of the note above of the + original transcriber describing his care to try to duplicate the + original 1887 edition as to typography and punctuation vagaries, + no changes have been made in this ascii text file. However, in + the Latin-1 file and this html file, present standards are + followed and the several French and Spanish words have been + given their proper accents. + + Part II, The Country of the Saints, deals much with the Mormon Church. + + + + +A STUDY IN SCARLET. + + + + + +PART I. + +(_Being a reprint from the reminiscences of_ JOHN H. WATSON, M.D., _late +of the Army Medical Department._) [2] + + + + +CHAPTER I. MR. SHERLOCK HOLMES. + + +IN the year 1878 I took my degree of Doctor of Medicine of the +University of London, and proceeded to Netley to go through the course +prescribed for surgeons in the army. Having completed my studies there, +I was duly attached to the Fifth Northumberland Fusiliers as Assistant +Surgeon. The regiment was stationed in India at the time, and before +I could join it, the second Afghan war had broken out. On landing at +Bombay, I learned that my corps had advanced through the passes, and +was already deep in the enemy's country. I followed, however, with many +other officers who were in the same situation as myself, and succeeded +in reaching Candahar in safety, where I found my regiment, and at once +entered upon my new duties. + +The campaign brought honours and promotion to many, but for me it had +nothing but misfortune and disaster. I was removed from my brigade and +attached to the Berkshires, with whom I served at the fatal battle of +Maiwand. There I was struck on the shoulder by a Jezail bullet, which +shattered the bone and grazed the subclavian artery. I should have +fallen into the hands of the murderous Ghazis had it not been for the +devotion and courage shown by Murray, my orderly, who threw me across a +pack-horse, and succeeded in bringing me safely to the British lines. + +Worn with pain, and weak from the prolonged hardships which I had +undergone, I was removed, with a great train of wounded sufferers, to +the base hospital at Peshawar. Here I rallied, and had already improved +so far as to be able to walk about the wards, and even to bask a little +upon the verandah, when I was struck down by enteric fever, that curse +of our Indian possessions. For months my life was despaired of, and +when at last I came to myself and became convalescent, I was so weak and +emaciated that a medical board determined that not a day should be lost +in sending me back to England. I was dispatched, accordingly, in the +troopship "Orontes," and landed a month later on Portsmouth jetty, with +my health irretrievably ruined, but with permission from a paternal +government to spend the next nine months in attempting to improve it. + +I had neither kith nor kin in England, and was therefore as free as +air--or as free as an income of eleven shillings and sixpence a day will +permit a man to be. Under such circumstances, I naturally gravitated to +London, that great cesspool into which all the loungers and idlers of +the Empire are irresistibly drained. There I stayed for some time at +a private hotel in the Strand, leading a comfortless, meaningless +existence, and spending such money as I had, considerably more freely +than I ought. So alarming did the state of my finances become, that +I soon realized that I must either leave the metropolis and rusticate +somewhere in the country, or that I must make a complete alteration in +my style of living. Choosing the latter alternative, I began by making +up my mind to leave the hotel, and to take up my quarters in some less +pretentious and less expensive domicile. + +On the very day that I had come to this conclusion, I was standing at +the Criterion Bar, when some one tapped me on the shoulder, and turning +round I recognized young Stamford, who had been a dresser under me at +Barts. The sight of a friendly face in the great wilderness of London is +a pleasant thing indeed to a lonely man. In old days Stamford had never +been a particular crony of mine, but now I hailed him with enthusiasm, +and he, in his turn, appeared to be delighted to see me. In the +exuberance of my joy, I asked him to lunch with me at the Holborn, and +we started off together in a hansom. + +"Whatever have you been doing with yourself, Watson?" he asked in +undisguised wonder, as we rattled through the crowded London streets. +"You are as thin as a lath and as brown as a nut." + +I gave him a short sketch of my adventures, and had hardly concluded it +by the time that we reached our destination. + +"Poor devil!" he said, commiseratingly, after he had listened to my +misfortunes. "What are you up to now?" + +"Looking for lodgings." [3] I answered. "Trying to solve the problem +as to whether it is possible to get comfortable rooms at a reasonable +price." + +"That's a strange thing," remarked my companion; "you are the second man +to-day that has used that expression to me." + +"And who was the first?" I asked. + +"A fellow who is working at the chemical laboratory up at the hospital. +He was bemoaning himself this morning because he could not get someone +to go halves with him in some nice rooms which he had found, and which +were too much for his purse." + +"By Jove!" I cried, "if he really wants someone to share the rooms and +the expense, I am the very man for him. I should prefer having a partner +to being alone." + +Young Stamford looked rather strangely at me over his wine-glass. "You +don't know Sherlock Holmes yet," he said; "perhaps you would not care +for him as a constant companion." + +"Why, what is there against him?" + +"Oh, I didn't say there was anything against him. He is a little queer +in his ideas--an enthusiast in some branches of science. As far as I +know he is a decent fellow enough." + +"A medical student, I suppose?" said I. + +"No--I have no idea what he intends to go in for. I believe he is well +up in anatomy, and he is a first-class chemist; but, as far as I know, +he has never taken out any systematic medical classes. His studies are +very desultory and eccentric, but he has amassed a lot of out-of-the way +knowledge which would astonish his professors." + +"Did you never ask him what he was going in for?" I asked. + +"No; he is not a man that it is easy to draw out, though he can be +communicative enough when the fancy seizes him." + +"I should like to meet him," I said. "If I am to lodge with anyone, I +should prefer a man of studious and quiet habits. I am not strong +enough yet to stand much noise or excitement. I had enough of both in +Afghanistan to last me for the remainder of my natural existence. How +could I meet this friend of yours?" + +"He is sure to be at the laboratory," returned my companion. "He either +avoids the place for weeks, or else he works there from morning to +night. If you like, we shall drive round together after luncheon." + +"Certainly," I answered, and the conversation drifted away into other +channels. + +As we made our way to the hospital after leaving the Holborn, Stamford +gave me a few more particulars about the gentleman whom I proposed to +take as a fellow-lodger. + +"You mustn't blame me if you don't get on with him," he said; "I know +nothing more of him than I have learned from meeting him occasionally in +the laboratory. You proposed this arrangement, so you must not hold me +responsible." + +"If we don't get on it will be easy to part company," I answered. "It +seems to me, Stamford," I added, looking hard at my companion, "that you +have some reason for washing your hands of the matter. Is this fellow's +temper so formidable, or what is it? Don't be mealy-mouthed about it." + +"It is not easy to express the inexpressible," he answered with a laugh. +"Holmes is a little too scientific for my tastes--it approaches to +cold-bloodedness. I could imagine his giving a friend a little pinch of +the latest vegetable alkaloid, not out of malevolence, you understand, +but simply out of a spirit of inquiry in order to have an accurate idea +of the effects. To do him justice, I think that he would take it himself +with the same readiness. He appears to have a passion for definite and +exact knowledge." + +"Very right too." + +"Yes, but it may be pushed to excess. When it comes to beating the +subjects in the dissecting-rooms with a stick, it is certainly taking +rather a bizarre shape." + +"Beating the subjects!" + +"Yes, to verify how far bruises may be produced after death. I saw him +at it with my own eyes." + +This eBook is for the use of anyone anywhere at no cost and with +almost no restrictions whatsoever. You may copy it, give it away or +re-use it under the terms of the Project Gutenberg License included +with this eBook or online at www.gutenberg.org + + +Title: A Study In Scarlet + +Author: Arthur Conan Doyle + +Posting Date: July 12, 2008 [EBook #244] +Release Date: April, 1995 +[Last updated: February 17, 2013] + +Language: English + + +*** START OF THIS PROJECT GUTENBERG EBOOK A STUDY IN SCARLET *** + + + + +Produced by Roger Squires + + + + + +A STUDY IN SCARLET. + +By A. Conan Doyle + +[1] + + + + Original Transcriber's Note: This etext is prepared directly + from an 1887 edition, and care has been taken to duplicate the + original exactly, including typographical and punctuation + vagaries. + + Additions to the text include adding the underscore character to + indicate italics, and textual end-notes in square braces. + + Project Gutenberg Editor's Note: In reproofing and moving old PG + files such as this to the present PG directory system it is the + policy to reformat the text to conform to present PG Standards. + In this case however, in consideration of the note above of the + original transcriber describing his care to try to duplicate the + original 1887 edition as to typography and punctuation vagaries, + no changes have been made in this ascii text file. However, in + the Latin-1 file and this html file, present standards are + followed and the several French and Spanish words have been + given their proper accents. + + Part II, The Country of the Saints, deals much with the Mormon Church. + + + + +A STUDY IN SCARLET. + + + + + +PART I. + +(_Being a reprint from the reminiscences of_ JOHN H. WATSON, M.D., _late +of the Army Medical Department._) [2] + + + + +CHAPTER I. MR. SHERLOCK HOLMES. + + +IN the year 1878 I took my degree of Doctor of Medicine of the +University of London, and proceeded to Netley to go through the course +prescribed for surgeons in the army. Having completed my studies there, +I was duly attached to the Fifth Northumberland Fusiliers as Assistant +Surgeon. The regiment was stationed in India at the time, and before +I could join it, the second Afghan war had broken out. On landing at +Bombay, I learned that my corps had advanced through the passes, and +was already deep in the enemy's country. I followed, however, with many +other officers who were in the same situation as myself, and succeeded +in reaching Candahar in safety, where I found my regiment, and at once +entered upon my new duties. + +The campaign brought honours and promotion to many, but for me it had +nothing but misfortune and disaster. I was removed from my brigade and +attached to the Berkshires, with whom I served at the fatal battle of +Maiwand. There I was struck on the shoulder by a Jezail bullet, which +shattered the bone and grazed the subclavian artery. I should have +fallen into the hands of the murderous Ghazis had it not been for the +devotion and courage shown by Murray, my orderly, who threw me across a +pack-horse, and succeeded in bringing me safely to the British lines. + +Worn with pain, and weak from the prolonged hardships which I had +undergone, I was removed, with a great train of wounded sufferers, to +the base hospital at Peshawar. Here I rallied, and had already improved +so far as to be able to walk about the wards, and even to bask a little +upon the verandah, when I was struck down by enteric fever, that curse +of our Indian possessions. For months my life was despaired of, and +when at last I came to myself and became convalescent, I was so weak and +emaciated that a medical board determined that not a day should be lost +in sending me back to England. I was dispatched, accordingly, in the +troopship "Orontes," and landed a month later on Portsmouth jetty, with +my health irretrievably ruined, but with permission from a paternal +government to spend the next nine months in attempting to improve it. + +I had neither kith nor kin in England, and was therefore as free as +air--or as free as an income of eleven shillings and sixpence a day will +permit a man to be. Under such circumstances, I naturally gravitated to +London, that great cesspool into which all the loungers and idlers of +the Empire are irresistibly drained. There I stayed for some time at +a private hotel in the Strand, leading a comfortless, meaningless +existence, and spending such money as I had, considerably more freely +than I ought. So alarming did the state of my finances become, that +I soon realized that I must either leave the metropolis and rusticate +somewhere in the country, or that I must make a complete alteration in +my style of living. Choosing the latter alternative, I began by making +up my mind to leave the hotel, and to take up my quarters in some less +pretentious and less expensive domicile. + +On the very day that I had come to this conclusion, I was standing at +the Criterion Bar, when some one tapped me on the shoulder, and turning +round I recognized young Stamford, who had been a dresser under me at +Barts. The sight of a friendly face in the great wilderness of London is +a pleasant thing indeed to a lonely man. In old days Stamford had never +been a particular crony of mine, but now I hailed him with enthusiasm, +and he, in his turn, appeared to be delighted to see me. In the +exuberance of my joy, I asked him to lunch with me at the Holborn, and +we started off together in a hansom. + +"Whatever have you been doing with yourself, Watson?" he asked in +undisguised wonder, as we rattled through the crowded London streets. +"You are as thin as a lath and as brown as a nut." + +I gave him a short sketch of my adventures, and had hardly concluded it +by the time that we reached our destination. + +"Poor devil!" he said, commiseratingly, after he had listened to my +misfortunes. "What are you up to now?" + +"Looking for lodgings." [3] I answered. "Trying to solve the problem +as to whether it is possible to get comfortable rooms at a reasonable +price." + +"That's a strange thing," remarked my companion; "you are the second man +to-day that has used that expression to me." + +"And who was the first?" I asked. + +"A fellow who is working at the chemical laboratory up at the hospital. +He was bemoaning himself this morning because he could not get someone +to go halves with him in some nice rooms which he had found, and which +were too much for his purse." + +"By Jove!" I cried, "if he really wants someone to share the rooms and +the expense, I am the very man for him. I should prefer having a partner +to being alone." + +Young Stamford looked rather strangely at me over his wine-glass. "You +don't know Sherlock Holmes yet," he said; "perhaps you would not care +for him as a constant companion." + +"Why, what is there against him?" + +"Oh, I didn't say there was anything against him. He is a little queer +in his ideas--an enthusiast in some branches of science. As far as I +know he is a decent fellow enough." + +"A medical student, I suppose?" said I. + +"No--I have no idea what he intends to go in for. I believe he is well +up in anatomy, and he is a first-class chemist; but, as far as I know, +he has never taken out any systematic medical classes. His studies are +very desultory and eccentric, but he has amassed a lot of out-of-the way +knowledge which would astonish his professors." + +"Did you never ask him what he was going in for?" I asked. + +"No; he is not a man that it is easy to draw out, though he can be +communicative enough when the fancy seizes him." + +"I should like to meet him," I said. "If I am to lodge with anyone, I +should prefer a man of studious and quiet habits. I am not strong +enough yet to stand much noise or excitement. I had enough of both in +Afghanistan to last me for the remainder of my natural existence. How +could I meet this friend of yours?" + +"He is sure to be at the laboratory," returned my companion. "He either +avoids the place for weeks, or else he works there from morning to +night. If you like, we shall drive round together after luncheon." + +"Certainly," I answered, and the conversation drifted away into other +channels. + +As we made our way to the hospital after leaving the Holborn, Stamford +gave me a few more particulars about the gentleman whom I proposed to +take as a fellow-lodger. + +"You mustn't blame me if you don't get on with him," he said; "I know +nothing more of him than I have learned from meeting him occasionally in +the laboratory. You proposed this arrangement, so you must not hold me +responsible." + +"If we don't get on it will be easy to part company," I answered. "It +seems to me, Stamford," I added, looking hard at my companion, "that you +have some reason for washing your hands of the matter. Is this fellow's +temper so formidable, or what is it? Don't be mealy-mouthed about it." + +"It is not easy to express the inexpressible," he answered with a laugh. +"Holmes is a little too scientific for my tastes--it approaches to +cold-bloodedness. I could imagine his giving a friend a little pinch of +the latest vegetable alkaloid, not out of malevolence, you understand, +but simply out of a spirit of inquiry in order to have an accurate idea +of the effects. To do him justice, I think that he would take it himself +with the same readiness. He appears to have a passion for definite and +exact knowledge." + +"Very right too." + +"Yes, but it may be pushed to excess. When it comes to beating the +subjects in the dissecting-rooms with a stick, it is certainly taking +rather a bizarre shape." + +"Beating the subjects!" + +"Yes, to verify how far bruises may be produced after death. I saw him +at it with my own eyes." + +This eBook is for the use of anyone anywhere at no cost and with +almost no restrictions whatsoever. You may copy it, give it away or +re-use it under the terms of the Project Gutenberg License included +with this eBook or online at www.gutenberg.org + + +Title: A Study In Scarlet + +Author: Arthur Conan Doyle + +Posting Date: July 12, 2008 [EBook #244] +Release Date: April, 1995 +[Last updated: February 17, 2013] + +Language: English + + +*** START OF THIS PROJECT GUTENBERG EBOOK A STUDY IN SCARLET *** + + + + +Produced by Roger Squires + + + + + +A STUDY IN SCARLET. + +By A. Conan Doyle + +[1] + + + + Original Transcriber's Note: This etext is prepared directly + from an 1887 edition, and care has been taken to duplicate the + original exactly, including typographical and punctuation + vagaries. + + Additions to the text include adding the underscore character to + indicate italics, and textual end-notes in square braces. + + Project Gutenberg Editor's Note: In reproofing and moving old PG + files such as this to the present PG directory system it is the + policy to reformat the text to conform to present PG Standards. + In this case however, in consideration of the note above of the + original transcriber describing his care to try to duplicate the + original 1887 edition as to typography and punctuation vagaries, + no changes have been made in this ascii text file. However, in + the Latin-1 file and this html file, present standards are + followed and the several French and Spanish words have been + given their proper accents. + + Part II, The Country of the Saints, deals much with the Mormon Church. + + + + +A STUDY IN SCARLET. + + + + + +PART I. + +(_Being a reprint from the reminiscences of_ JOHN H. WATSON, M.D., _late +of the Army Medical Department._) [2] + + + + +CHAPTER I. MR. SHERLOCK HOLMES. + + +IN the year 1878 I took my degree of Doctor of Medicine of the +University of London, and proceeded to Netley to go through the course +prescribed for surgeons in the army. Having completed my studies there, +I was duly attached to the Fifth Northumberland Fusiliers as Assistant +Surgeon. The regiment was stationed in India at the time, and before +I could join it, the second Afghan war had broken out. On landing at +Bombay, I learned that my corps had advanced through the passes, and +was already deep in the enemy's country. I followed, however, with many +other officers who were in the same situation as myself, and succeeded +in reaching Candahar in safety, where I found my regiment, and at once +entered upon my new duties. + +The campaign brought honours and promotion to many, but for me it had +nothing but misfortune and disaster. I was removed from my brigade and +attached to the Berkshires, with whom I served at the fatal battle of +Maiwand. There I was struck on the shoulder by a Jezail bullet, which +shattered the bone and grazed the subclavian artery. I should have +fallen into the hands of the murderous Ghazis had it not been for the +devotion and courage shown by Murray, my orderly, who threw me across a +pack-horse, and succeeded in bringing me safely to the British lines. + +Worn with pain, and weak from the prolonged hardships which I had +undergone, I was removed, with a great train of wounded sufferers, to +the base hospital at Peshawar. Here I rallied, and had already improved +so far as to be able to walk about the wards, and even to bask a little +upon the verandah, when I was struck down by enteric fever, that curse +of our Indian possessions. For months my life was despaired of, and +when at last I came to myself and became convalescent, I was so weak and +emaciated that a medical board determined that not a day should be lost +in sending me back to England. I was dispatched, accordingly, in the +troopship "Orontes," and landed a month later on Portsmouth jetty, with +my health irretrievably ruined, but with permission from a paternal +government to spend the next nine months in attempting to improve it. + +I had neither kith nor kin in England, and was therefore as free as +air--or as free as an income of eleven shillings and sixpence a day will +permit a man to be. Under such circumstances, I naturally gravitated to +London, that great cesspool into which all the loungers and idlers of +the Empire are irresistibly drained. There I stayed for some time at +a private hotel in the Strand, leading a comfortless, meaningless +existence, and spending such money as I had, considerably more freely +than I ought. So alarming did the state of my finances become, that +I soon realized that I must either leave the metropolis and rusticate +somewhere in the country, or that I must make a complete alteration in +my style of living. Choosing the latter alternative, I began by making +up my mind to leave the hotel, and to take up my quarters in some less +pretentious and less expensive domicile. + +On the very day that I had come to this conclusion, I was standing at +the Criterion Bar, when some one tapped me on the shoulder, and turning +round I recognized young Stamford, who had been a dresser under me at +Barts. The sight of a friendly face in the great wilderness of London is +a pleasant thing indeed to a lonely man. In old days Stamford had never +been a particular crony of mine, but now I hailed him with enthusiasm, +and he, in his turn, appeared to be delighted to see me. In the +exuberance of my joy, I asked him to lunch with me at the Holborn, and +we started off together in a hansom. + +"Whatever have you been doing with yourself, Watson?" he asked in +undisguised wonder, as we rattled through the crowded London streets. +"You are as thin as a lath and as brown as a nut." + +I gave him a short sketch of my adventures, and had hardly concluded it +by the time that we reached our destination. + +"Poor devil!" he said, commiseratingly, after he had listened to my +misfortunes. "What are you up to now?" + +"Looking for lodgings." [3] I answered. "Trying to solve the problem +as to whether it is possible to get comfortable rooms at a reasonable +price." + +"That's a strange thing," remarked my companion; "you are the second man +to-day that has used that expression to me." + +"And who was the first?" I asked. + +"A fellow who is working at the chemical laboratory up at the hospital. +He was bemoaning himself this morning because he could not get someone +to go halves with him in some nice rooms which he had found, and which +were too much for his purse." + +"By Jove!" I cried, "if he really wants someone to share the rooms and +the expense, I am the very man for him. I should prefer having a partner +to being alone." + +Young Stamford looked rather strangely at me over his wine-glass. "You +don't know Sherlock Holmes yet," he said; "perhaps you would not care +for him as a constant companion." + +"Why, what is there against him?" + +"Oh, I didn't say there was anything against him. He is a little queer +in his ideas--an enthusiast in some branches of science. As far as I +know he is a decent fellow enough." + +"A medical student, I suppose?" said I. + +"No--I have no idea what he intends to go in for. I believe he is well +up in anatomy, and he is a first-class chemist; but, as far as I know, +he has never taken out any systematic medical classes. His studies are +very desultory and eccentric, but he has amassed a lot of out-of-the way +knowledge which would astonish his professors." + +"Did you never ask him what he was going in for?" I asked. + +"No; he is not a man that it is easy to draw out, though he can be +communicative enough when the fancy seizes him." + +"I should like to meet him," I said. "If I am to lodge with anyone, I +should prefer a man of studious and quiet habits. I am not strong +enough yet to stand much noise or excitement. I had enough of both in +Afghanistan to last me for the remainder of my natural existence. How +could I meet this friend of yours?" + +"He is sure to be at the laboratory," returned my companion. "He either +avoids the place for weeks, or else he works there from morning to +night. If you like, we shall drive round together after luncheon." + +"Certainly," I answered, and the conversation drifted away into other +channels. + +As we made our way to the hospital after leaving the Holborn, Stamford +gave me a few more particulars about the gentleman whom I proposed to +take as a fellow-lodger. + +"You mustn't blame me if you don't get on with him," he said; "I know +nothing more of him than I have learned from meeting him occasionally in +the laboratory. You proposed this arrangement, so you must not hold me +responsible." + +"If we don't get on it will be easy to part company," I answered. "It +seems to me, Stamford," I added, looking hard at my companion, "that you +have some reason for washing your hands of the matter. Is this fellow's +temper so formidable, or what is it? Don't be mealy-mouthed about it." + +"It is not easy to express the inexpressible," he answered with a laugh. +"Holmes is a little too scientific for my tastes--it approaches to +cold-bloodedness. I could imagine his giving a friend a little pinch of +the latest vegetable alkaloid, not out of malevolence, you understand, +but simply out of a spirit of inquiry in order to have an accurate idea +of the effects. To do him justice, I think that he would take it himself +with the same readiness. He appears to have a passion for definite and +exact knowledge." + +"Very right too." + +"Yes, but it may be pushed to excess. When it comes to beating the +subjects in the dissecting-rooms with a stick, it is certainly taking +rather a bizarre shape." + +"Beating the subjects!" + +"Yes, to verify how far bruises may be produced after death. I saw him +at it with my own eyes." + +This eBook is for the use of anyone anywhere at no cost and with +almost no restrictions whatsoever. You may copy it, give it away or +re-use it under the terms of the Project Gutenberg License included +with this eBook or online at www.gutenberg.org + + +Title: A Study In Scarlet + +Author: Arthur Conan Doyle + +Posting Date: July 12, 2008 [EBook #244] +Release Date: April, 1995 +[Last updated: February 17, 2013] + +Language: English + + +*** START OF THIS PROJECT GUTENBERG EBOOK A STUDY IN SCARLET *** + + + + +Produced by Roger Squires + + + + + +A STUDY IN SCARLET. + +By A. Conan Doyle + +[1] + + + + Original Transcriber's Note: This etext is prepared directly + from an 1887 edition, and care has been taken to duplicate the + original exactly, including typographical and punctuation + vagaries. + + Additions to the text include adding the underscore character to + indicate italics, and textual end-notes in square braces. + + Project Gutenberg Editor's Note: In reproofing and moving old PG + files such as this to the present PG directory system it is the + policy to reformat the text to conform to present PG Standards. + In this case however, in consideration of the note above of the + original transcriber describing his care to try to duplicate the + original 1887 edition as to typography and punctuation vagaries, + no changes have been made in this ascii text file. However, in + the Latin-1 file and this html file, present standards are + followed and the several French and Spanish words have been + given their proper accents. + + Part II, The Country of the Saints, deals much with the Mormon Church. + + + + +A STUDY IN SCARLET. + + + + + +PART I. + +(_Being a reprint from the reminiscences of_ JOHN H. WATSON, M.D., _late +of the Army Medical Department._) [2] + + + + +CHAPTER I. MR. SHERLOCK HOLMES. + + +IN the year 1878 I took my degree of Doctor of Medicine of the +University of London, and proceeded to Netley to go through the course +prescribed for surgeons in the army. Having completed my studies there, +I was duly attached to the Fifth Northumberland Fusiliers as Assistant +Surgeon. The regiment was stationed in India at the time, and before +I could join it, the second Afghan war had broken out. On landing at +Bombay, I learned that my corps had advanced through the passes, and +was already deep in the enemy's country. I followed, however, with many +other officers who were in the same situation as myself, and succeeded +in reaching Candahar in safety, where I found my regiment, and at once +entered upon my new duties. + +The campaign brought honours and promotion to many, but for me it had +nothing but misfortune and disaster. I was removed from my brigade and +attached to the Berkshires, with whom I served at the fatal battle of +Maiwand. There I was struck on the shoulder by a Jezail bullet, which +shattered the bone and grazed the subclavian artery. I should have +fallen into the hands of the murderous Ghazis had it not been for the +devotion and courage shown by Murray, my orderly, who threw me across a +pack-horse, and succeeded in bringing me safely to the British lines. + +Worn with pain, and weak from the prolonged hardships which I had +undergone, I was removed, with a great train of wounded sufferers, to +the base hospital at Peshawar. Here I rallied, and had already improved +so far as to be able to walk about the wards, and even to bask a little +upon the verandah, when I was struck down by enteric fever, that curse +of our Indian possessions. For months my life was despaired of, and +when at last I came to myself and became convalescent, I was so weak and +emaciated that a medical board determined that not a day should be lost +in sending me back to England. I was dispatched, accordingly, in the +troopship "Orontes," and landed a month later on Portsmouth jetty, with +my health irretrievably ruined, but with permission from a paternal +government to spend the next nine months in attempting to improve it. + +I had neither kith nor kin in England, and was therefore as free as +air--or as free as an income of eleven shillings and sixpence a day will +permit a man to be. Under such circumstances, I naturally gravitated to +London, that great cesspool into which all the loungers and idlers of +the Empire are irresistibly drained. There I stayed for some time at +a private hotel in the Strand, leading a comfortless, meaningless +existence, and spending such money as I had, considerably more freely +than I ought. So alarming did the state of my finances become, that +I soon realized that I must either leave the metropolis and rusticate +somewhere in the country, or that I must make a complete alteration in +my style of living. Choosing the latter alternative, I began by making +up my mind to leave the hotel, and to take up my quarters in some less +pretentious and less expensive domicile. + +On the very day that I had come to this conclusion, I was standing at +the Criterion Bar, when some one tapped me on the shoulder, and turning +round I recognized young Stamford, who had been a dresser under me at +Barts. The sight of a friendly face in the great wilderness of London is +a pleasant thing indeed to a lonely man. In old days Stamford had never +been a particular crony of mine, but now I hailed him with enthusiasm, +and he, in his turn, appeared to be delighted to see me. In the +exuberance of my joy, I asked him to lunch with me at the Holborn, and +we started off together in a hansom. + +"Whatever have you been doing with yourself, Watson?" he asked in +undisguised wonder, as we rattled through the crowded London streets. +"You are as thin as a lath and as brown as a nut." + +I gave him a short sketch of my adventures, and had hardly concluded it +by the time that we reached our destination. + +"Poor devil!" he said, commiseratingly, after he had listened to my +misfortunes. "What are you up to now?" + +"Looking for lodgings." [3] I answered. "Trying to solve the problem +as to whether it is possible to get comfortable rooms at a reasonable +price." + +"That's a strange thing," remarked my companion; "you are the second man +to-day that has used that expression to me." + +"And who was the first?" I asked. + +"A fellow who is working at the chemical laboratory up at the hospital. +He was bemoaning himself this morning because he could not get someone +to go halves with him in some nice rooms which he had found, and which +were too much for his purse." + +"By Jove!" I cried, "if he really wants someone to share the rooms and +the expense, I am the very man for him. I should prefer having a partner +to being alone." + +Young Stamford looked rather strangely at me over his wine-glass. "You +don't know Sherlock Holmes yet," he said; "perhaps you would not care +for him as a constant companion." + +"Why, what is there against him?" + +"Oh, I didn't say there was anything against him. He is a little queer +in his ideas--an enthusiast in some branches of science. As far as I +know he is a decent fellow enough." + +"A medical student, I suppose?" said I. + +"No--I have no idea what he intends to go in for. I believe he is well +up in anatomy, and he is a first-class chemist; but, as far as I know, +he has never taken out any systematic medical classes. His studies are +very desultory and eccentric, but he has amassed a lot of out-of-the way +knowledge which would astonish his professors." + +"Did you never ask him what he was going in for?" I asked. + +"No; he is not a man that it is easy to draw out, though he can be +communicative enough when the fancy seizes him." + +"I should like to meet him," I said. "If I am to lodge with anyone, I +should prefer a man of studious and quiet habits. I am not strong +enough yet to stand much noise or excitement. I had enough of both in +Afghanistan to last me for the remainder of my natural existence. How +could I meet this friend of yours?" + +"He is sure to be at the laboratory," returned my companion. "He either +avoids the place for weeks, or else he works there from morning to +night. If you like, we shall drive round together after luncheon." + +"Certainly," I answered, and the conversation drifted away into other +channels. + +As we made our way to the hospital after leaving the Holborn, Stamford +gave me a few more particulars about the gentleman whom I proposed to +take as a fellow-lodger. + +"You mustn't blame me if you don't get on with him," he said; "I know +nothing more of him than I have learned from meeting him occasionally in +the laboratory. You proposed this arrangement, so you must not hold me +responsible." + +"If we don't get on it will be easy to part company," I answered. "It +seems to me, Stamford," I added, looking hard at my companion, "that you +have some reason for washing your hands of the matter. Is this fellow's +temper so formidable, or what is it? Don't be mealy-mouthed about it." + +"It is not easy to express the inexpressible," he answered with a laugh. +"Holmes is a little too scientific for my tastes--it approaches to +cold-bloodedness. I could imagine his giving a friend a little pinch of +the latest vegetable alkaloid, not out of malevolence, you understand, +but simply out of a spirit of inquiry in order to have an accurate idea +of the effects. To do him justice, I think that he would take it himself +with the same readiness. He appears to have a passion for definite and +exact knowledge." + +"Very right too." + +"Yes, but it may be pushed to excess. When it comes to beating the +subjects in the dissecting-rooms with a stick, it is certainly taking +rather a bizarre shape." + +"Beating the subjects!" + +"Yes, to verify how far bruises may be produced after death. I saw him +at it with my own eyes." + +This eBook is for the use of anyone anywhere at no cost and with +almost no restrictions whatsoever. You may copy it, give it away or +re-use it under the terms of the Project Gutenberg License included +with this eBook or online at www.gutenberg.org + + +Title: A Study In Scarlet + +Author: Arthur Conan Doyle + +Posting Date: July 12, 2008 [EBook #244] +Release Date: April, 1995 +[Last updated: February 17, 2013] + +Language: English + + +*** START OF THIS PROJECT GUTENBERG EBOOK A STUDY IN SCARLET *** + + + + +Produced by Roger Squires + + + + + +A STUDY IN SCARLET. + +By A. Conan Doyle + +[1] + + + + Original Transcriber's Note: This etext is prepared directly + from an 1887 edition, and care has been taken to duplicate the + original exactly, including typographical and punctuation + vagaries. + + Additions to the text include adding the underscore character to + indicate italics, and textual end-notes in square braces. + + Project Gutenberg Editor's Note: In reproofing and moving old PG + files such as this to the present PG directory system it is the + policy to reformat the text to conform to present PG Standards. + In this case however, in consideration of the note above of the + original transcriber describing his care to try to duplicate the + original 1887 edition as to typography and punctuation vagaries, + no changes have been made in this ascii text file. However, in + the Latin-1 file and this html file, present standards are + followed and the several French and Spanish words have been + given their proper accents. + + Part II, The Country of the Saints, deals much with the Mormon Church. + + + + +A STUDY IN SCARLET. + + + + + +PART I. + +(_Being a reprint from the reminiscences of_ JOHN H. WATSON, M.D., _late +of the Army Medical Department._) [2] + + + + +CHAPTER I. MR. SHERLOCK HOLMES. + + +IN the year 1878 I took my degree of Doctor of Medicine of the +University of London, and proceeded to Netley to go through the course +prescribed for surgeons in the army. Having completed my studies there, +I was duly attached to the Fifth Northumberland Fusiliers as Assistant +Surgeon. The regiment was stationed in India at the time, and before +I could join it, the second Afghan war had broken out. On landing at +Bombay, I learned that my corps had advanced through the passes, and +was already deep in the enemy's country. I followed, however, with many +other officers who were in the same situation as myself, and succeeded +in reaching Candahar in safety, where I found my regiment, and at once +entered upon my new duties. + +The campaign brought honours and promotion to many, but for me it had +nothing but misfortune and disaster. I was removed from my brigade and +attached to the Berkshires, with whom I served at the fatal battle of +Maiwand. There I was struck on the shoulder by a Jezail bullet, which +shattered the bone and grazed the subclavian artery. I should have +fallen into the hands of the murderous Ghazis had it not been for the +devotion and courage shown by Murray, my orderly, who threw me across a +pack-horse, and succeeded in bringing me safely to the British lines. + +Worn with pain, and weak from the prolonged hardships which I had +undergone, I was removed, with a great train of wounded sufferers, to +the base hospital at Peshawar. Here I rallied, and had already improved +so far as to be able to walk about the wards, and even to bask a little +upon the verandah, when I was struck down by enteric fever, that curse +of our Indian possessions. For months my life was despaired of, and +when at last I came to myself and became convalescent, I was so weak and +emaciated that a medical board determined that not a day should be lost +in sending me back to England. I was dispatched, accordingly, in the +troopship "Orontes," and landed a month later on Portsmouth jetty, with +my health irretrievably ruined, but with permission from a paternal +government to spend the next nine months in attempting to improve it. + +I had neither kith nor kin in England, and was therefore as free as +air--or as free as an income of eleven shillings and sixpence a day will +permit a man to be. Under such circumstances, I naturally gravitated to +London, that great cesspool into which all the loungers and idlers of +the Empire are irresistibly drained. There I stayed for some time at +a private hotel in the Strand, leading a comfortless, meaningless +existence, and spending such money as I had, considerably more freely +than I ought. So alarming did the state of my finances become, that +I soon realized that I must either leave the metropolis and rusticate +somewhere in the country, or that I must make a complete alteration in +my style of living. Choosing the latter alternative, I began by making +up my mind to leave the hotel, and to take up my quarters in some less +pretentious and less expensive domicile. + +On the very day that I had come to this conclusion, I was standing at +the Criterion Bar, when some one tapped me on the shoulder, and turning +round I recognized young Stamford, who had been a dresser under me at +Barts. The sight of a friendly face in the great wilderness of London is +a pleasant thing indeed to a lonely man. In old days Stamford had never +been a particular crony of mine, but now I hailed him with enthusiasm, +and he, in his turn, appeared to be delighted to see me. In the +exuberance of my joy, I asked him to lunch with me at the Holborn, and +we started off together in a hansom. + +"Whatever have you been doing with yourself, Watson?" he asked in +undisguised wonder, as we rattled through the crowded London streets. +"You are as thin as a lath and as brown as a nut." + +I gave him a short sketch of my adventures, and had hardly concluded it +by the time that we reached our destination. + +"Poor devil!" he said, commiseratingly, after he had listened to my +misfortunes. "What are you up to now?" + +"Looking for lodgings." [3] I answered. "Trying to solve the problem +as to whether it is possible to get comfortable rooms at a reasonable +price." + +"That's a strange thing," remarked my companion; "you are the second man +to-day that has used that expression to me." + +"And who was the first?" I asked. + +"A fellow who is working at the chemical laboratory up at the hospital. +He was bemoaning himself this morning because he could not get someone +to go halves with him in some nice rooms which he had found, and which +were too much for his purse." + +"By Jove!" I cried, "if he really wants someone to share the rooms and +the expense, I am the very man for him. I should prefer having a partner +to being alone." + +Young Stamford looked rather strangely at me over his wine-glass. "You +don't know Sherlock Holmes yet," he said; "perhaps you would not care +for him as a constant companion." + +"Why, what is there against him?" + +"Oh, I didn't say there was anything against him. He is a little queer +in his ideas--an enthusiast in some branches of science. As far as I +know he is a decent fellow enough." + +"A medical student, I suppose?" said I. + +"No--I have no idea what he intends to go in for. I believe he is well +up in anatomy, and he is a first-class chemist; but, as far as I know, +he has never taken out any systematic medical classes. His studies are +very desultory and eccentric, but he has amassed a lot of out-of-the way +knowledge which would astonish his professors." + +"Did you never ask him what he was going in for?" I asked. + +"No; he is not a man that it is easy to draw out, though he can be +communicative enough when the fancy seizes him." + +"I should like to meet him," I said. "If I am to lodge with anyone, I +should prefer a man of studious and quiet habits. I am not strong +enough yet to stand much noise or excitement. I had enough of both in +Afghanistan to last me for the remainder of my natural existence. How +could I meet this friend of yours?" + +"He is sure to be at the laboratory," returned my companion. "He either +avoids the place for weeks, or else he works there from morning to +night. If you like, we shall drive round together after luncheon." + +"Certainly," I answered, and the conversation drifted away into other +channels. + +As we made our way to the hospital after leaving the Holborn, Stamford +gave me a few more particulars about the gentleman whom I proposed to +take as a fellow-lodger. + +"You mustn't blame me if you don't get on with him," he said; "I know +nothing more of him than I have learned from meeting him occasionally in +the laboratory. You proposed this arrangement, so you must not hold me +responsible." + +"If we don't get on it will be easy to part company," I answered. "It +seems to me, Stamford," I added, looking hard at my companion, "that you +have some reason for washing your hands of the matter. Is this fellow's +temper so formidable, or what is it? Don't be mealy-mouthed about it." + +"It is not easy to express the inexpressible," he answered with a laugh. +"Holmes is a little too scientific for my tastes--it approaches to +cold-bloodedness. I could imagine his giving a friend a little pinch of +the latest vegetable alkaloid, not out of malevolence, you understand, +but simply out of a spirit of inquiry in order to have an accurate idea +of the effects. To do him justice, I think that he would take it himself +with the same readiness. He appears to have a passion for definite and +exact knowledge." + +"Very right too." + +"Yes, but it may be pushed to excess. When it comes to beating the +subjects in the dissecting-rooms with a stick, it is certainly taking +rather a bizarre shape." + +"Beating the subjects!" + +"Yes, to verify how far bruises may be produced after death. I saw him +at it with my own eyes." + +This eBook is for the use of anyone anywhere at no cost and with +almost no restrictions whatsoever. You may copy it, give it away or +re-use it under the terms of the Project Gutenberg License included +with this eBook or online at www.gutenberg.org + + +Title: A Study In Scarlet + +Author: Arthur Conan Doyle + +Posting Date: July 12, 2008 [EBook #244] +Release Date: April, 1995 +[Last updated: February 17, 2013] + +Language: English + + +*** START OF THIS PROJECT GUTENBERG EBOOK A STUDY IN SCARLET *** + + + + +Produced by Roger Squires + + + + + +A STUDY IN SCARLET. + +By A. Conan Doyle + +[1] + + + + Original Transcriber's Note: This etext is prepared directly + from an 1887 edition, and care has been taken to duplicate the + original exactly, including typographical and punctuation + vagaries. + + Additions to the text include adding the underscore character to + indicate italics, and textual end-notes in square braces. + + Project Gutenberg Editor's Note: In reproofing and moving old PG + files such as this to the present PG directory system it is the + policy to reformat the text to conform to present PG Standards. + In this case however, in consideration of the note above of the + original transcriber describing his care to try to duplicate the + original 1887 edition as to typography and punctuation vagaries, + no changes have been made in this ascii text file. However, in + the Latin-1 file and this html file, present standards are + followed and the several French and Spanish words have been + given their proper accents. + + Part II, The Country of the Saints, deals much with the Mormon Church. + + + + +A STUDY IN SCARLET. + + + + + +PART I. + +(_Being a reprint from the reminiscences of_ JOHN H. WATSON, M.D., _late +of the Army Medical Department._) [2] + + + + +CHAPTER I. MR. SHERLOCK HOLMES. + + +IN the year 1878 I took my degree of Doctor of Medicine of the +University of London, and proceeded to Netley to go through the course +prescribed for surgeons in the army. Having completed my studies there, +I was duly attached to the Fifth Northumberland Fusiliers as Assistant +Surgeon. The regiment was stationed in India at the time, and before +I could join it, the second Afghan war had broken out. On landing at +Bombay, I learned that my corps had advanced through the passes, and +was already deep in the enemy's country. I followed, however, with many +other officers who were in the same situation as myself, and succeeded +in reaching Candahar in safety, where I found my regiment, and at once +entered upon my new duties. + +The campaign brought honours and promotion to many, but for me it had +nothing but misfortune and disaster. I was removed from my brigade and +attached to the Berkshires, with whom I served at the fatal battle of +Maiwand. There I was struck on the shoulder by a Jezail bullet, which +shattered the bone and grazed the subclavian artery. I should have +fallen into the hands of the murderous Ghazis had it not been for the +devotion and courage shown by Murray, my orderly, who threw me across a +pack-horse, and succeeded in bringing me safely to the British lines. + +Worn with pain, and weak from the prolonged hardships which I had +undergone, I was removed, with a great train of wounded sufferers, to +the base hospital at Peshawar. Here I rallied, and had already improved +so far as to be able to walk about the wards, and even to bask a little +upon the verandah, when I was struck down by enteric fever, that curse +of our Indian possessions. For months my life was despaired of, and +when at last I came to myself and became convalescent, I was so weak and +emaciated that a medical board determined that not a day should be lost +in sending me back to England. I was dispatched, accordingly, in the +troopship "Orontes," and landed a month later on Portsmouth jetty, with +my health irretrievably ruined, but with permission from a paternal +government to spend the next nine months in attempting to improve it. + +I had neither kith nor kin in England, and was therefore as free as +air--or as free as an income of eleven shillings and sixpence a day will +permit a man to be. Under such circumstances, I naturally gravitated to +London, that great cesspool into which all the loungers and idlers of +the Empire are irresistibly drained. There I stayed for some time at +a private hotel in the Strand, leading a comfortless, meaningless +existence, and spending such money as I had, considerably more freely +than I ought. So alarming did the state of my finances become, that +I soon realized that I must either leave the metropolis and rusticate +somewhere in the country, or that I must make a complete alteration in +my style of living. Choosing the latter alternative, I began by making +up my mind to leave the hotel, and to take up my quarters in some less +pretentious and less expensive domicile. + +On the very day that I had come to this conclusion, I was standing at +the Criterion Bar, when some one tapped me on the shoulder, and turning +round I recognized young Stamford, who had been a dresser under me at +Barts. The sight of a friendly face in the great wilderness of London is +a pleasant thing indeed to a lonely man. In old days Stamford had never +been a particular crony of mine, but now I hailed him with enthusiasm, +and he, in his turn, appeared to be delighted to see me. In the +exuberance of my joy, I asked him to lunch with me at the Holborn, and +we started off together in a hansom. + +"Whatever have you been doing with yourself, Watson?" he asked in +undisguised wonder, as we rattled through the crowded London streets. +"You are as thin as a lath and as brown as a nut." + +I gave him a short sketch of my adventures, and had hardly concluded it +by the time that we reached our destination. + +"Poor devil!" he said, commiseratingly, after he had listened to my +misfortunes. "What are you up to now?" + +"Looking for lodgings." [3] I answered. "Trying to solve the problem +as to whether it is possible to get comfortable rooms at a reasonable +price." + +"That's a strange thing," remarked my companion; "you are the second man +to-day that has used that expression to me." + +"And who was the first?" I asked. + +"A fellow who is working at the chemical laboratory up at the hospital. +He was bemoaning himself this morning because he could not get someone +to go halves with him in some nice rooms which he had found, and which +were too much for his purse." + +"By Jove!" I cried, "if he really wants someone to share the rooms and +the expense, I am the very man for him. I should prefer having a partner +to being alone." + +Young Stamford looked rather strangely at me over his wine-glass. "You +don't know Sherlock Holmes yet," he said; "perhaps you would not care +for him as a constant companion." + +"Why, what is there against him?" + +"Oh, I didn't say there was anything against him. He is a little queer +in his ideas--an enthusiast in some branches of science. As far as I +know he is a decent fellow enough." + +"A medical student, I suppose?" said I. + +"No--I have no idea what he intends to go in for. I believe he is well +up in anatomy, and he is a first-class chemist; but, as far as I know, +he has never taken out any systematic medical classes. His studies are +very desultory and eccentric, but he has amassed a lot of out-of-the way +knowledge which would astonish his professors." + +"Did you never ask him what he was going in for?" I asked. + +"No; he is not a man that it is easy to draw out, though he can be +communicative enough when the fancy seizes him." + +"I should like to meet him," I said. "If I am to lodge with anyone, I +should prefer a man of studious and quiet habits. I am not strong +enough yet to stand much noise or excitement. I had enough of both in +Afghanistan to last me for the remainder of my natural existence. How +could I meet this friend of yours?" + +"He is sure to be at the laboratory," returned my companion. "He either +avoids the place for weeks, or else he works there from morning to +night. If you like, we shall drive round together after luncheon." + +"Certainly," I answered, and the conversation drifted away into other +channels. + +As we made our way to the hospital after leaving the Holborn, Stamford +gave me a few more particulars about the gentleman whom I proposed to +take as a fellow-lodger. + +"You mustn't blame me if you don't get on with him," he said; "I know +nothing more of him than I have learned from meeting him occasionally in +the laboratory. You proposed this arrangement, so you must not hold me +responsible." + +"If we don't get on it will be easy to part company," I answered. "It +seems to me, Stamford," I added, looking hard at my companion, "that you +have some reason for washing your hands of the matter. Is this fellow's +temper so formidable, or what is it? Don't be mealy-mouthed about it." + +"It is not easy to express the inexpressible," he answered with a laugh. +"Holmes is a little too scientific for my tastes--it approaches to +cold-bloodedness. I could imagine his giving a friend a little pinch of +the latest vegetable alkaloid, not out of malevolence, you understand, +but simply out of a spirit of inquiry in order to have an accurate idea +of the effects. To do him justice, I think that he would take it himself +with the same readiness. He appears to have a passion for definite and +exact knowledge." + +"Very right too." + +"Yes, but it may be pushed to excess. When it comes to beating the +subjects in the dissecting-rooms with a stick, it is certainly taking +rather a bizarre shape." + +"Beating the subjects!" + +"Yes, to verify how far bruises may be produced after death. I saw him +at it with my own eyes." + +This eBook is for the use of anyone anywhere at no cost and with +almost no restrictions whatsoever. You may copy it, give it away or +re-use it under the terms of the Project Gutenberg License included +with this eBook or online at www.gutenberg.org + + +Title: A Study In Scarlet + +Author: Arthur Conan Doyle + +Posting Date: July 12, 2008 [EBook #244] +Release Date: April, 1995 +[Last updated: February 17, 2013] + +Language: English + + +*** START OF THIS PROJECT GUTENBERG EBOOK A STUDY IN SCARLET *** + + + + +Produced by Roger Squires + + + + + +A STUDY IN SCARLET. + +By A. Conan Doyle + +[1] + + + + Original Transcriber's Note: This etext is prepared directly + from an 1887 edition, and care has been taken to duplicate the + original exactly, including typographical and punctuation + vagaries. + + Additions to the text include adding the underscore character to + indicate italics, and textual end-notes in square braces. + + Project Gutenberg Editor's Note: In reproofing and moving old PG + files such as this to the present PG directory system it is the + policy to reformat the text to conform to present PG Standards. + In this case however, in consideration of the note above of the + original transcriber describing his care to try to duplicate the + original 1887 edition as to typography and punctuation vagaries, + no changes have been made in this ascii text file. However, in + the Latin-1 file and this html file, present standards are + followed and the several French and Spanish words have been + given their proper accents. + + Part II, The Country of the Saints, deals much with the Mormon Church. + + + + +A STUDY IN SCARLET. + + + + + +PART I. + +(_Being a reprint from the reminiscences of_ JOHN H. WATSON, M.D., _late +of the Army Medical Department._) [2] + + + + +CHAPTER I. MR. SHERLOCK HOLMES. + + +IN the year 1878 I took my degree of Doctor of Medicine of the +University of London, and proceeded to Netley to go through the course +prescribed for surgeons in the army. Having completed my studies there, +I was duly attached to the Fifth Northumberland Fusiliers as Assistant +Surgeon. The regiment was stationed in India at the time, and before +I could join it, the second Afghan war had broken out. On landing at +Bombay, I learned that my corps had advanced through the passes, and +was already deep in the enemy's country. I followed, however, with many +other officers who were in the same situation as myself, and succeeded +in reaching Candahar in safety, where I found my regiment, and at once +entered upon my new duties. + +The campaign brought honours and promotion to many, but for me it had +nothing but misfortune and disaster. I was removed from my brigade and +attached to the Berkshires, with whom I served at the fatal battle of +Maiwand. There I was struck on the shoulder by a Jezail bullet, which +shattered the bone and grazed the subclavian artery. I should have +fallen into the hands of the murderous Ghazis had it not been for the +devotion and courage shown by Murray, my orderly, who threw me across a +pack-horse, and succeeded in bringing me safely to the British lines. + +Worn with pain, and weak from the prolonged hardships which I had +undergone, I was removed, with a great train of wounded sufferers, to +the base hospital at Peshawar. Here I rallied, and had already improved +so far as to be able to walk about the wards, and even to bask a little +upon the verandah, when I was struck down by enteric fever, that curse +of our Indian possessions. For months my life was despaired of, and +when at last I came to myself and became convalescent, I was so weak and +emaciated that a medical board determined that not a day should be lost +in sending me back to England. I was dispatched, accordingly, in the +troopship "Orontes," and landed a month later on Portsmouth jetty, with +my health irretrievably ruined, but with permission from a paternal +government to spend the next nine months in attempting to improve it. + +I had neither kith nor kin in England, and was therefore as free as +air--or as free as an income of eleven shillings and sixpence a day will +permit a man to be. Under such circumstances, I naturally gravitated to +London, that great cesspool into which all the loungers and idlers of +the Empire are irresistibly drained. There I stayed for some time at +a private hotel in the Strand, leading a comfortless, meaningless +existence, and spending such money as I had, considerably more freely +than I ought. So alarming did the state of my finances become, that +I soon realized that I must either leave the metropolis and rusticate +somewhere in the country, or that I must make a complete alteration in +my style of living. Choosing the latter alternative, I began by making +up my mind to leave the hotel, and to take up my quarters in some less +pretentious and less expensive domicile. + +On the very day that I had come to this conclusion, I was standing at +the Criterion Bar, when some one tapped me on the shoulder, and turning +round I recognized young Stamford, who had been a dresser under me at +Barts. The sight of a friendly face in the great wilderness of London is +a pleasant thing indeed to a lonely man. In old days Stamford had never +been a particular crony of mine, but now I hailed him with enthusiasm, +and he, in his turn, appeared to be delighted to see me. In the +exuberance of my joy, I asked him to lunch with me at the Holborn, and +we started off together in a hansom. + +"Whatever have you been doing with yourself, Watson?" he asked in +undisguised wonder, as we rattled through the crowded London streets. +"You are as thin as a lath and as brown as a nut." + +I gave him a short sketch of my adventures, and had hardly concluded it +by the time that we reached our destination. + +"Poor devil!" he said, commiseratingly, after he had listened to my +misfortunes. "What are you up to now?" + +"Looking for lodgings." [3] I answered. "Trying to solve the problem +as to whether it is possible to get comfortable rooms at a reasonable +price." + +"That's a strange thing," remarked my companion; "you are the second man +to-day that has used that expression to me." + +"And who was the first?" I asked. + +"A fellow who is working at the chemical laboratory up at the hospital. +He was bemoaning himself this morning because he could not get someone +to go halves with him in some nice rooms which he had found, and which +were too much for his purse." + +"By Jove!" I cried, "if he really wants someone to share the rooms and +the expense, I am the very man for him. I should prefer having a partner +to being alone." + +Young Stamford looked rather strangely at me over his wine-glass. "You +don't know Sherlock Holmes yet," he said; "perhaps you would not care +for him as a constant companion." + +"Why, what is there against him?" + +"Oh, I didn't say there was anything against him. He is a little queer +in his ideas--an enthusiast in some branches of science. As far as I +know he is a decent fellow enough." + +"A medical student, I suppose?" said I. + +"No--I have no idea what he intends to go in for. I believe he is well +up in anatomy, and he is a first-class chemist; but, as far as I know, +he has never taken out any systematic medical classes. His studies are +very desultory and eccentric, but he has amassed a lot of out-of-the way +knowledge which would astonish his professors." + +"Did you never ask him what he was going in for?" I asked. + +"No; he is not a man that it is easy to draw out, though he can be +communicative enough when the fancy seizes him." + +"I should like to meet him," I said. "If I am to lodge with anyone, I +should prefer a man of studious and quiet habits. I am not strong +enough yet to stand much noise or excitement. I had enough of both in +Afghanistan to last me for the remainder of my natural existence. How +could I meet this friend of yours?" + +"He is sure to be at the laboratory," returned my companion. "He either +avoids the place for weeks, or else he works there from morning to +night. If you like, we shall drive round together after luncheon." + +"Certainly," I answered, and the conversation drifted away into other +channels. + +As we made our way to the hospital after leaving the Holborn, Stamford +gave me a few more particulars about the gentleman whom I proposed to +take as a fellow-lodger. + +"You mustn't blame me if you don't get on with him," he said; "I know +nothing more of him than I have learned from meeting him occasionally in +the laboratory. You proposed this arrangement, so you must not hold me +responsible." + +"If we don't get on it will be easy to part company," I answered. "It +seems to me, Stamford," I added, looking hard at my companion, "that you +have some reason for washing your hands of the matter. Is this fellow's +temper so formidable, or what is it? Don't be mealy-mouthed about it." + +"It is not easy to express the inexpressible," he answered with a laugh. +"Holmes is a little too scientific for my tastes--it approaches to +cold-bloodedness. I could imagine his giving a friend a little pinch of +the latest vegetable alkaloid, not out of malevolence, you understand, +but simply out of a spirit of inquiry in order to have an accurate idea +of the effects. To do him justice, I think that he would take it himself +with the same readiness. He appears to have a passion for definite and +exact knowledge." + +"Very right too." + +"Yes, but it may be pushed to excess. When it comes to beating the +subjects in the dissecting-rooms with a stick, it is certainly taking +rather a bizarre shape." + +"Beating the subjects!" + +"Yes, to verify how far bruises may be produced after death. I saw him +at it with my own eyes." + +"And yet you say he is not a medical student?" + + +abcdef +"No. Heaven knows what the objects of his studies are. But here we +are, and you must form your own impressions about him." As he spoke, we +turned down a narrow lane and passed through a small side-door, which +opened into a wing of the great hospital. It was familiar ground to me, +and I needed no guiding as we ascended the bleak stone staircase and +made our way down the long corridor with its vista of whitewashed +wall and dun-coloured doors. Near the further end a low arched passage +branched away from it and led to the chemical laboratory. + +This was a lofty chamber, lined and littered with countless bottles. +Broad, low tables were scattered about, which bristled with retorts, +test-tubes, and little Bunsen lamps, with their blue flickering flames. +There was only one student in the room, who was bending over a distant +table absorbed in his work. At the sound of our steps he glanced round +and sprang to his feet with a cry of pleasure. "I've found it! I've +found it," he shouted to my companion, running towards us with a +test-tube in his hand. "I have found a re-agent which is precipitated +by hoemoglobin, [4] and by nothing else." Had he discovered a gold mine, +greater delight could not have shone upon his features. + +"Dr. Watson, Mr. Sherlock Holmes," said Stamford, introducing us. + +"How are you?" he said cordially, gripping my hand with a strength +for which I should hardly have given him credit. "You have been in +Afghanistan, I perceive." + +"How on earth did you know that?" I asked in astonishment. + +"Never mind," said he, chuckling to himself. "The question now is about +hoemoglobin. No doubt you see the significance of this discovery of +mine?" + +"It is interesting, chemically, no doubt," I answered, "but +practically----" + +"Why, man, it is the most practical medico-legal discovery for years. +Don't you see that it gives us an infallible test for blood stains. Come +over here now!" He seized me by the coat-sleeve in his eagerness, and +drew me over to the table at which he had been working. "Let us have +some fresh blood," he said, digging a long bodkin into his finger, and +drawing off the resulting drop of blood in a chemical pipette. "Now, I +add this small quantity of blood to a litre of water. You perceive that +the resulting mixture has the appearance of pure water. The proportion +of blood cannot be more than one in a million. I have no doubt, however, +that we shall be able to obtain the characteristic reaction." As he +spoke, he threw into the vessel a few white crystals, and then added +some drops of a transparent fluid. In an instant the contents assumed a +dull mahogany colour, and a brownish dust was precipitated to the bottom +of the glass jar. + +"Ha! ha!" he cried, clapping his hands, and looking as delighted as a +child with a new toy. "What do you think of that?" + +"It seems to be a very delicate test," I remarked. + +"Beautiful! beautiful! The old Guiacum test was very clumsy and +uncertain. So is the microscopic examination for blood corpuscles. The +latter is valueless if the stains are a few hours old. Now, this appears +to act as well whether the blood is old or new. Had this test been +invented, there are hundreds of men now walking the earth who would long +ago have paid the penalty of their crimes." + +"Indeed!" I murmured. + +"Criminal cases are continually hinging upon that one point. A man is +suspected of a crime months perhaps after it has been committed. His +linen or clothes are examined, and brownish stains discovered upon them. +Are they blood stains, or mud stains, or rust stains, or fruit stains, +or what are they? That is a question which has puzzled many an expert, +and why? Because there was no reliable test. Now we have the Sherlock +Holmes' test, and there will no longer be any difficulty." + +His eyes fairly glittered as he spoke, and he put his hand over his +heart and bowed as if to some applauding crowd conjured up by his +imagination. + +"You are to be congratulated," I remarked, considerably surprised at his +enthusiasm. + +"There was the case of Von Bischoff at Frankfort last year. He would +certainly have been hung had this test been in existence. Then there was +Mason of Bradford, and the notorious Muller, and Lefevre of Montpellier, +and Samson of New Orleans. I could name a score of cases in which it +would have been decisive." + +"You seem to be a walking calendar of crime," said Stamford with a +laugh. "You might start a paper on those lines. Call it the 'Police News +of the Past.'" + +"Very interesting reading it might be made, too," remarked Sherlock +Holmes, sticking a small piece of plaster over the prick on his finger. +"I have to be careful," he continued, turning to me with a smile, "for I +dabble with poisons a good deal." He held out his hand as he spoke, and +I noticed that it was all mottled over with similar pieces of plaster, +and discoloured with strong acids. + +"We came here on business," said Stamford, sitting down on a high +three-legged stool, and pushing another one in my direction with +his foot. "My friend here wants to take diggings, and as you were +complaining that you could get no one to go halves with you, I thought +that I had better bring you together." + +Sherlock Holmes seemed delighted at the idea of sharing his rooms with +me. "I have my eye on a suite in Baker Street," he said, "which would +suit us down to the ground. You don't mind the smell of strong tobacco, +I hope?" + +"I always smoke 'ship's' myself," I answered. + +"That's good enough. I generally have chemicals about, and occasionally +do experiments. Would that annoy you?" + +"By no means." + +"Let me see--what are my other shortcomings. I get in the dumps at +times, and don't open my mouth for days on end. You must not think I am +sulky when I do that. Just let me alone, and I'll soon be right. What +have you to confess now? It's just as well for two fellows to know the +worst of one another before they begin to live together." + +I laughed at this cross-examination. "I keep a bull pup," I said, "and +I object to rows because my nerves are shaken, and I get up at all sorts +of ungodly hours, and I am extremely lazy. I have another set of vices +when I'm well, but those are the principal ones at present." + +"Do you include violin-playing in your category of rows?" he asked, +anxiously. + +"It depends on the player," I answered. "A well-played violin is a treat +for the gods--a badly-played one----" + +"Oh, that's all right," he cried, with a merry laugh. "I think we may +consider the thing as settled--that is, if the rooms are agreeable to +you." + +"When shall we see them?" + +"Call for me here at noon to-morrow, and we'll go together and settle +everything," he answered. + +"All right--noon exactly," said I, shaking his hand. + +We left him working among his chemicals, and we walked together towards +my hotel. + +"By the way," I asked suddenly, stopping and turning upon Stamford, "how +the deuce did he know that I had come from Afghanistan?" + +My companion smiled an enigmatical smile. "That's just his little +peculiarity," he said. "A good many people have wanted to know how he +finds things out." + +"Oh! a mystery is it?" I cried, rubbing my hands. "This is very piquant. +I am much obliged to you for bringing us together. 'The proper study of +mankind is man,' you know." + +"You must study him, then," Stamford said, as he bade me good-bye. +"You'll find him a knotty problem, though. I'll wager he learns more +about you than you about him. Good-bye." + +"Good-bye," I answered, and strolled on to my hotel, considerably +interested in my new acquaintance. + + + + +CHAPTER II. THE SCIENCE OF DEDUCTION. + + +WE met next day as he had arranged, and inspected the rooms at No. 221B, +[5] Baker Street, of which he had spoken at our meeting. They +consisted of a couple of comfortable bed-rooms and a single large +airy sitting-room, cheerfully furnished, and illuminated by two broad +windows. So desirable in every way were the apartments, and so moderate +did the terms seem when divided between us, that the bargain was +concluded upon the spot, and we at once entered into possession. +That very evening I moved my things round from the hotel, and on the +following morning Sherlock Holmes followed me with several boxes and +portmanteaus. For a day or two we were busily employed in unpacking and +laying out our property to the best advantage. That done, we +gradually began to settle down and to accommodate ourselves to our new +surroundings. + +Holmes was certainly not a difficult man to live with. He was quiet +in his ways, and his habits were regular. It was rare for him to be +up after ten at night, and he had invariably breakfasted and gone out +before I rose in the morning. Sometimes he spent his day at the chemical +laboratory, sometimes in the dissecting-rooms, and occasionally in long +walks, which appeared to take him into the lowest portions of the City. +Nothing could exceed his energy when the working fit was upon him; but +now and again a reaction would seize him, and for days on end he would +lie upon the sofa in the sitting-room, hardly uttering a word or moving +a muscle from morning to night. On these occasions I have noticed such +a dreamy, vacant expression in his eyes, that I might have suspected him +of being addicted to the use of some narcotic, had not the temperance +and cleanliness of his whole life forbidden such a notion. + +As the weeks went by, my interest in him and my curiosity as to his +aims in life, gradually deepened and increased. His very person and +appearance were such as to strike the attention of the most casual +observer. In height he was rather over six feet, and so excessively +lean that he seemed to be considerably taller. His eyes were sharp and +piercing, save during those intervals of torpor to which I have alluded; +and his thin, hawk-like nose gave his whole expression an air of +alertness and decision. His chin, too, had the prominence and squareness +which mark the man of determination. His hands were invariably +blotted with ink and stained with chemicals, yet he was possessed of +extraordinary delicacy of touch, as I frequently had occasion to observe +when I watched him manipulating his fragile philosophical instruments. + +The reader may set me down as a hopeless busybody, when I confess how +much this man stimulated my curiosity, and how often I endeavoured +to break through the reticence which he showed on all that concerned +himself. Before pronouncing judgment, however, be it remembered, how +objectless was my life, and how little there was to engage my attention. +My health forbade me from venturing out unless the weather was +exceptionally genial, and I had no friends who would call upon me and +break the monotony of my daily existence. Under these circumstances, I +eagerly hailed the little mystery which hung around my companion, and +spent much of my time in endeavouring to unravel it. + +He was not studying medicine. He had himself, in reply to a question, +confirmed Stamford's opinion upon that point. Neither did he appear to +have pursued any course of reading which might fit him for a degree in +science or any other recognized portal which would give him an entrance +into the learned world. Yet his zeal for certain studies was remarkable, +and within eccentric limits his knowledge was so extraordinarily ample +and minute that his observations have fairly astounded me. Surely no man +would work so hard or attain such precise information unless he had some +definite end in view. Desultory readers are seldom remarkable for the +exactness of their learning. No man burdens his mind with small matters +unless he has some very good reason for doing so. + +His ignorance was as remarkable as his knowledge. Of contemporary +literature, philosophy and politics he appeared to know next to nothing. +Upon my quoting Thomas Carlyle, he inquired in the naivest way who he +might be and what he had done. My surprise reached a climax, however, +when I found incidentally that he was ignorant of the Copernican Theory +and of the composition of the Solar System. That any civilized human +being in this nineteenth century should not be aware that the earth +travelled round the sun appeared to be to me such an extraordinary fact +that I could hardly realize it. + +"You appear to be astonished," he said, smiling at my expression of +surprise. "Now that I do know it I shall do my best to forget it." + +"To forget it!" + +"You see," he explained, "I consider that a man's brain originally is +like a little empty attic, and you have to stock it with such furniture +as you choose. A fool takes in all the lumber of every sort that he +comes across, so that the knowledge which might be useful to him gets +crowded out, or at best is jumbled up with a lot of other things so that +he has a difficulty in laying his hands upon it. Now the skilful workman +is very careful indeed as to what he takes into his brain-attic. He will +have nothing but the tools which may help him in doing his work, but of +these he has a large assortment, and all in the most perfect order. It +is a mistake to think that that little room has elastic walls and can +distend to any extent. Depend upon it there comes a time when for every +addition of knowledge you forget something that you knew before. It is +of the highest importance, therefore, not to have useless facts elbowing +out the useful ones." + diff --git a/userland/ripgrep/tests/data/sherlock.Z b/userland/ripgrep/tests/data/sherlock.Z new file mode 100644 index 0000000..9ffa928 Binary files /dev/null and b/userland/ripgrep/tests/data/sherlock.Z differ diff --git a/userland/ripgrep/tests/data/sherlock.br b/userland/ripgrep/tests/data/sherlock.br new file mode 100644 index 0000000..7d0a961 --- /dev/null +++ b/userland/ripgrep/tests/data/sherlock.br @@ -0,0 +1,2 @@ +n ÅÔ-_£.ž ÂàèîcM–•´…Yî4”þÿÖöYa ¤-L²O(Â8Àsn^Gwш!,µ +KDŒ”/7‡ëthƒÌ ´Ò]jÊåÔÁ³E_;dàrF©QsÈ/:DIVB}¢T7„‚ѵÝãH2ŒçÑ)óÄM[uÁõði‰¿Ðã50Ú®Y6°÷ƒÒ“®è¬%ר_ ©U by“4¥†Ï ‚!&Ðgé#í \ No newline at end of file diff --git a/userland/ripgrep/tests/data/sherlock.bz2 b/userland/ripgrep/tests/data/sherlock.bz2 new file mode 100644 index 0000000..e4a6454 Binary files /dev/null and b/userland/ripgrep/tests/data/sherlock.bz2 differ diff --git a/userland/ripgrep/tests/data/sherlock.gz b/userland/ripgrep/tests/data/sherlock.gz new file mode 100644 index 0000000..5629cbe Binary files /dev/null and b/userland/ripgrep/tests/data/sherlock.gz differ diff --git a/userland/ripgrep/tests/data/sherlock.lz4 b/userland/ripgrep/tests/data/sherlock.lz4 new file mode 100644 index 0000000..5913dbb Binary files /dev/null and b/userland/ripgrep/tests/data/sherlock.lz4 differ diff --git a/userland/ripgrep/tests/data/sherlock.lzma b/userland/ripgrep/tests/data/sherlock.lzma new file mode 100644 index 0000000..bfdf7fb Binary files /dev/null and b/userland/ripgrep/tests/data/sherlock.lzma differ diff --git a/userland/ripgrep/tests/data/sherlock.xz b/userland/ripgrep/tests/data/sherlock.xz new file mode 100644 index 0000000..39cca0f Binary files /dev/null and b/userland/ripgrep/tests/data/sherlock.xz differ diff --git a/userland/ripgrep/tests/data/sherlock.zst b/userland/ripgrep/tests/data/sherlock.zst new file mode 100644 index 0000000..c649ac9 Binary files /dev/null and b/userland/ripgrep/tests/data/sherlock.zst differ diff --git a/userland/ripgrep/tests/feature.rs b/userland/ripgrep/tests/feature.rs new file mode 100644 index 0000000..1424249 --- /dev/null +++ b/userland/ripgrep/tests/feature.rs @@ -0,0 +1,1174 @@ +use crate::hay::{SHERLOCK, SHERLOCK_CRLF}; +use crate::util::{Dir, TestCommand, sort_lines}; + +// See: https://github.com/BurntSushi/ripgrep/issues/1 +rgtest!(f1_sjis, |dir: Dir, mut cmd: TestCommand| { + dir.create_bytes( + "foo", + b"\x84Y\x84u\x84\x82\x84|\x84\x80\x84{ \x84V\x84\x80\x84|\x84}\x84\x83" + ); + cmd.arg("-Esjis").arg("Шерлок ХолмÑ"); + eqnice!("foo:Шерлок ХолмÑ\n", cmd.stdout()); +}); + +// See: https://github.com/BurntSushi/ripgrep/issues/1 +rgtest!(f1_utf16_auto, |dir: Dir, mut cmd: TestCommand| { + dir.create_bytes( + "foo", + b"\xff\xfe(\x045\x04@\x04;\x04>\x04:\x04 \x00%\x04>\x04;\x04<\x04A\x04" + ); + cmd.arg("Шерлок ХолмÑ"); + eqnice!("foo:Шерлок ХолмÑ\n", cmd.stdout()); +}); + +// See: https://github.com/BurntSushi/ripgrep/issues/1 +rgtest!(f1_utf16_explicit, |dir: Dir, mut cmd: TestCommand| { + dir.create_bytes( + "foo", + b"\xff\xfe(\x045\x04@\x04;\x04>\x04:\x04 \x00%\x04>\x04;\x04<\x04A\x04" + ); + cmd.arg("-Eutf-16le").arg("Шерлок ХолмÑ"); + eqnice!("foo:Шерлок ХолмÑ\n", cmd.stdout()); +}); + +// See: https://github.com/BurntSushi/ripgrep/issues/1 +rgtest!(f1_eucjp, |dir: Dir, mut cmd: TestCommand| { + dir.create_bytes( + "foo", + b"\xa7\xba\xa7\xd6\xa7\xe2\xa7\xdd\xa7\xe0\xa7\xdc \xa7\xb7\xa7\xe0\xa7\xdd\xa7\xde\xa7\xe3" + ); + cmd.arg("-Eeuc-jp").arg("Шерлок ХолмÑ"); + eqnice!("foo:Шерлок ХолмÑ\n", cmd.stdout()); +}); + +// See: https://github.com/BurntSushi/ripgrep/issues/1 +rgtest!(f1_unknown_encoding, |_: Dir, mut cmd: TestCommand| { + cmd.arg("-Efoobar").assert_non_empty_stderr(); +}); + +// See: https://github.com/BurntSushi/ripgrep/issues/1 +rgtest!(f1_replacement_encoding, |_: Dir, mut cmd: TestCommand| { + cmd.arg("-Ecsiso2022kr").assert_non_empty_stderr(); +}); + +// See: https://github.com/BurntSushi/ripgrep/issues/7 +rgtest!(f7, |dir: Dir, mut cmd: TestCommand| { + dir.create("sherlock", SHERLOCK); + dir.create("pat", "Sherlock\nHolmes"); + + let expected = "\ +For the Doctor Watsons of this world, as opposed to the Sherlock +Holmeses, success in the province of detective work must always +be, to a very large extent, the result of luck. Sherlock Holmes +"; + eqnice!(expected, cmd.arg("-fpat").arg("sherlock").stdout()); +}); + +// See: https://github.com/BurntSushi/ripgrep/issues/7 +rgtest!(f7_stdin, |dir: Dir, mut cmd: TestCommand| { + dir.create("sherlock", SHERLOCK); + + let expected = "\ +sherlock:For the Doctor Watsons of this world, as opposed to the Sherlock +sherlock:be, to a very large extent, the result of luck. Sherlock Holmes +"; + eqnice!(expected, cmd.arg("-f-").pipe(b"Sherlock")); +}); + +// See: https://github.com/BurntSushi/ripgrep/issues/20 +rgtest!(f20_no_filename, |dir: Dir, mut cmd: TestCommand| { + dir.create("sherlock", SHERLOCK); + cmd.arg("--no-filename"); + + let expected = "\ +For the Doctor Watsons of this world, as opposed to the Sherlock +be, to a very large extent, the result of luck. Sherlock Holmes +"; + eqnice!(expected, cmd.arg("--no-filename").arg("Sherlock").stdout()); +}); + +// See: https://github.com/BurntSushi/ripgrep/issues/34 +rgtest!(f34_only_matching, |dir: Dir, mut cmd: TestCommand| { + dir.create("sherlock", SHERLOCK); + + let expected = "\ +sherlock:Sherlock +sherlock:Sherlock +"; + eqnice!(expected, cmd.arg("-o").arg("Sherlock").stdout()); +}); + +// See: https://github.com/BurntSushi/ripgrep/issues/34 +rgtest!(f34_only_matching_line_column, |dir: Dir, mut cmd: TestCommand| { + dir.create("sherlock", SHERLOCK); + + let expected = "\ +sherlock:1:57:Sherlock +sherlock:3:49:Sherlock +"; + cmd.arg("-o").arg("--column").arg("-n").arg("Sherlock"); + eqnice!(expected, cmd.stdout()); +}); + +// See: https://github.com/BurntSushi/ripgrep/issues/45 +rgtest!(f45_relative_cwd, |dir: Dir, mut cmd: TestCommand| { + dir.create(".not-an-ignore", "foo\n/bar"); + dir.create_dir("bar"); + dir.create_dir("baz/bar"); + dir.create_dir("baz/baz/bar"); + dir.create("bar/test", "test"); + dir.create("baz/bar/test", "test"); + dir.create("baz/baz/bar/test", "test"); + dir.create("baz/foo", "test"); + dir.create("baz/test", "test"); + dir.create("foo", "test"); + dir.create("test", "test"); + + cmd.arg("-l").arg("test"); + + // First, get a baseline without applying ignore rules. + let expected = " +bar/test +baz/bar/test +baz/baz/bar/test +baz/foo +baz/test +foo +test +"; + eqnice!(sort_lines(expected), sort_lines(&cmd.stdout())); + + // Now try again with the ignore file activated. + cmd.arg("--ignore-file").arg(".not-an-ignore"); + let expected = " +baz/bar/test +baz/baz/bar/test +baz/test +test +"; + eqnice!(sort_lines(expected), sort_lines(&cmd.stdout())); + + // Now do it again, but inside the baz directory. Since the ignore file + // is interpreted relative to the CWD, this will cause the /bar anchored + // pattern to filter out baz/bar, which is a subtle difference between true + // parent ignore files and manually specified ignore files. + let mut cmd = dir.command(); + cmd.args(&["--ignore-file", "../.not-an-ignore", "-l", "test"]); + cmd.current_dir("baz"); + let expected = " +baz/bar/test +test +"; + eqnice!(sort_lines(expected), sort_lines(&cmd.stdout())); +}); + +// See: https://github.com/BurntSushi/ripgrep/issues/45 +rgtest!(f45_precedence_with_others, |dir: Dir, mut cmd: TestCommand| { + dir.create(".not-an-ignore", "*.log"); + dir.create(".ignore", "!imp.log"); + dir.create("imp.log", "test"); + dir.create("wat.log", "test"); + + cmd.arg("--ignore-file").arg(".not-an-ignore").arg("test"); + eqnice!("imp.log:test\n", cmd.stdout()); +}); + +// See: https://github.com/BurntSushi/ripgrep/issues/45 +rgtest!(f45_precedence_internal, |dir: Dir, mut cmd: TestCommand| { + dir.create(".not-an-ignore1", "*.log"); + dir.create(".not-an-ignore2", "!imp.log"); + dir.create("imp.log", "test"); + dir.create("wat.log", "test"); + + cmd.args(&[ + "--ignore-file", + ".not-an-ignore1", + "--ignore-file", + ".not-an-ignore2", + "test", + ]); + eqnice!("imp.log:test\n", cmd.stdout()); +}); + +// See: https://github.com/BurntSushi/ripgrep/issues/68 +rgtest!(f68_no_ignore_vcs, |dir: Dir, mut cmd: TestCommand| { + dir.create_dir(".git"); + dir.create(".gitignore", "foo"); + dir.create(".ignore", "bar"); + dir.create("foo", "test"); + dir.create("bar", "test"); + + eqnice!("foo:test\n", cmd.arg("--no-ignore-vcs").arg("test").stdout()); +}); + +// See: https://github.com/BurntSushi/ripgrep/issues/70 +rgtest!(f70_smart_case, |dir: Dir, mut cmd: TestCommand| { + dir.create("sherlock", SHERLOCK); + + let expected = "\ +sherlock:For the Doctor Watsons of this world, as opposed to the Sherlock +sherlock:be, to a very large extent, the result of luck. Sherlock Holmes +"; + eqnice!(expected, cmd.arg("-S").arg("sherlock").stdout()); +}); + +// See: https://github.com/BurntSushi/ripgrep/issues/89 +rgtest!(f89_files_with_matches, |dir: Dir, mut cmd: TestCommand| { + dir.create("sherlock", SHERLOCK); + + cmd.arg("--null").arg("--files-with-matches").arg("Sherlock"); + eqnice!("sherlock\x00", cmd.stdout()); +}); + +// See: https://github.com/BurntSushi/ripgrep/issues/89 +rgtest!(f89_files_without_match, |dir: Dir, mut cmd: TestCommand| { + dir.create("sherlock", SHERLOCK); + dir.create("file.py", "foo"); + + cmd.arg("--null").arg("--files-without-match").arg("Sherlock"); + eqnice!("file.py\x00", cmd.stdout()); +}); + +// See: https://github.com/BurntSushi/ripgrep/issues/89 +rgtest!(f89_count, |dir: Dir, mut cmd: TestCommand| { + dir.create("sherlock", SHERLOCK); + + cmd.arg("--null").arg("--count").arg("Sherlock"); + eqnice!("sherlock\x002\n", cmd.stdout()); +}); + +// See: https://github.com/BurntSushi/ripgrep/issues/89 +rgtest!(f89_files, |dir: Dir, mut cmd: TestCommand| { + dir.create("sherlock", SHERLOCK); + + eqnice!("sherlock\x00", cmd.arg("--null").arg("--files").stdout()); +}); + +// See: https://github.com/BurntSushi/ripgrep/issues/89 +rgtest!(f89_match, |dir: Dir, mut cmd: TestCommand| { + dir.create("sherlock", SHERLOCK); + + let expected = "\ +sherlock\x00For the Doctor Watsons of this world, as opposed to the Sherlock +sherlock\x00Holmeses, success in the province of detective work must always +sherlock\x00be, to a very large extent, the result of luck. Sherlock Holmes +sherlock\x00can extract a clew from a wisp of straw or a flake of cigar ash; +"; + eqnice!(expected, cmd.arg("--null").arg("-C1").arg("Sherlock").stdout()); +}); + +// See: https://github.com/BurntSushi/ripgrep/issues/109 +rgtest!(f109_max_depth, |dir: Dir, mut cmd: TestCommand| { + dir.create_dir("one"); + dir.create("one/pass", "far"); + dir.create_dir("one/too"); + dir.create("one/too/many", "far"); + + cmd.arg("--maxdepth").arg("2").arg("far"); + eqnice!("one/pass:far\n", cmd.stdout()); +}); + +// See: https://github.com/BurntSushi/ripgrep/issues/124 +rgtest!(f109_case_sensitive_part1, |dir: Dir, mut cmd: TestCommand| { + dir.create("foo", "tEsT"); + + cmd.arg("--smart-case").arg("--case-sensitive").arg("test").assert_err(); +}); + +// See: https://github.com/BurntSushi/ripgrep/issues/124 +rgtest!(f109_case_sensitive_part2, |dir: Dir, mut cmd: TestCommand| { + dir.create("foo", "tEsT"); + cmd.arg("--ignore-case").arg("--case-sensitive").arg("test").assert_err(); +}); + +// See: https://github.com/BurntSushi/ripgrep/issues/129 +rgtest!(f129_matches, |dir: Dir, mut cmd: TestCommand| { + dir.create("foo", "test\ntest abcdefghijklmnopqrstuvwxyz test"); + + let expected = "foo:test\nfoo:[Omitted long matching line]\n"; + eqnice!(expected, cmd.arg("-M26").arg("test").stdout()); +}); + +// See: https://github.com/BurntSushi/ripgrep/issues/129 +rgtest!(f129_context, |dir: Dir, mut cmd: TestCommand| { + dir.create("foo", "test\nabcdefghijklmnopqrstuvwxyz"); + + let expected = "foo:test\nfoo-[Omitted long context line]\n"; + eqnice!(expected, cmd.arg("-M20").arg("-C1").arg("test").stdout()); +}); + +// See: https://github.com/BurntSushi/ripgrep/issues/129 +rgtest!(f129_replace, |dir: Dir, mut cmd: TestCommand| { + dir.create("foo", "test\ntest abcdefghijklmnopqrstuvwxyz test"); + + let expected = "foo:foo\nfoo:[Omitted long line with 2 matches]\n"; + eqnice!(expected, cmd.arg("-M26").arg("-rfoo").arg("test").stdout()); +}); + +// See: https://github.com/BurntSushi/ripgrep/issues/159 +rgtest!(f159_max_count, |dir: Dir, mut cmd: TestCommand| { + dir.create("foo", "test\ntest"); + + eqnice!("foo:test\n", cmd.arg("-m1").arg("test").stdout()); +}); + +// See: https://github.com/BurntSushi/ripgrep/issues/159 +rgtest!(f159_max_count_zero, |dir: Dir, mut cmd: TestCommand| { + dir.create("foo", "test\ntest"); + + cmd.arg("-m0").arg("test").assert_err(); +}); + +// See: https://github.com/BurntSushi/ripgrep/issues/196 +rgtest!(f196_persistent_config, |dir: Dir, mut cmd: TestCommand| { + dir.create("sherlock", SHERLOCK); + cmd.arg("sherlock").arg("sherlock"); + + // Make sure we get no matches by default. + cmd.assert_err(); + + // Now add our config file, and make sure it impacts ripgrep. + dir.create(".ripgreprc", "--ignore-case"); + cmd.cmd().env("RIPGREP_CONFIG_PATH", ".ripgreprc"); + + let expected = "\ +For the Doctor Watsons of this world, as opposed to the Sherlock +be, to a very large extent, the result of luck. Sherlock Holmes +"; + eqnice!(expected, cmd.stdout()); +}); + +// See: https://github.com/BurntSushi/ripgrep/issues/243 +rgtest!(f243_column_line, |dir: Dir, mut cmd: TestCommand| { + dir.create("foo", "test"); + + eqnice!("foo:1:1:test\n", cmd.arg("--column").arg("test").stdout()); +}); + +// See: https://github.com/BurntSushi/ripgrep/issues/263 +rgtest!(f263_sort_files, |dir: Dir, mut cmd: TestCommand| { + dir.create("foo", "test"); + dir.create("abc", "test"); + dir.create("zoo", "test"); + dir.create("bar", "test"); + + let expected = "abc:test\nbar:test\nfoo:test\nzoo:test\n"; + eqnice!(expected, cmd.arg("--sort-files").arg("test").stdout()); +}); + +// See: https://github.com/BurntSushi/ripgrep/issues/263 +rgtest!(f263_sort_files_reverse, |dir: Dir, mut cmd: TestCommand| { + dir.create("foo", "test"); + dir.create("abc", "test"); + dir.create("zoo", "test"); + dir.create("bar", "test"); + + let expected = "zoo:test\nfoo:test\nbar:test\nabc:test\n"; + eqnice!(expected, cmd.arg("--sortr=path").arg("test").stdout()); +}); + +// See: https://github.com/BurntSushi/ripgrep/issues/275 +rgtest!(f275_pathsep, |dir: Dir, mut cmd: TestCommand| { + dir.create_dir("foo"); + dir.create("foo/bar", "test"); + + cmd.arg("test").arg("--path-separator").arg("Z"); + eqnice!("fooZbar:test\n", cmd.stdout()); +}); + +// See: https://github.com/BurntSushi/ripgrep/issues/362 +rgtest!(f362_dfa_size_limit, |dir: Dir, mut cmd: TestCommand| { + dir.create("sherlock", SHERLOCK); + + // This should fall back to the nfa engine but should still produce the + // expected result. + cmd.arg("--dfa-size-limit").arg("10").arg(r"For\s").arg("sherlock"); + + let expected = "\ +For the Doctor Watsons of this world, as opposed to the Sherlock +"; + eqnice!(expected, cmd.stdout()); +}); + +// See: https://github.com/BurntSushi/ripgrep/issues/362 +rgtest!(f362_exceeds_regex_size_limit, |dir: Dir, mut cmd: TestCommand| { + // --regex-size-limit doesn't apply to PCRE2. + if dir.is_pcre2() { + return; + } + cmd.arg("--regex-size-limit").arg("10K").arg(r"[0-9]\w+").assert_err(); +}); + +// See: https://github.com/BurntSushi/ripgrep/issues/362 +#[cfg(target_pointer_width = "32")] +rgtest!( + f362_u64_to_narrow_usize_overflow, + |dir: Dir, mut cmd: TestCommand| { + // --dfa-size-limit doesn't apply to PCRE2. + if dir.is_pcre2() { + return; + } + dir.create_size("foo", 1000000); + + // 2^35 * 2^20 is ok for u64, but not for usize + cmd.arg("--dfa-size-limit").arg("34359738368M").arg("--files"); + cmd.assert_err(); + } +); + +// See: https://github.com/BurntSushi/ripgrep/issues/411 +rgtest!( + f411_single_threaded_search_stats, + |dir: Dir, mut cmd: TestCommand| { + dir.create("sherlock", SHERLOCK); + + let lines = cmd.arg("-j1").arg("--stats").arg("Sherlock").stdout(); + assert!(lines.contains("Sherlock")); + assert!(lines.contains("2 matched lines")); + assert!(lines.contains("1 files contained matches")); + assert!(lines.contains("1 files searched")); + assert!(lines.contains("seconds")); + } +); + +rgtest!(f411_parallel_search_stats, |dir: Dir, mut cmd: TestCommand| { + dir.create("sherlock_1", SHERLOCK); + dir.create("sherlock_2", SHERLOCK); + + let lines = cmd.arg("-j2").arg("--stats").arg("Sherlock").stdout(); + assert!(lines.contains("4 matched lines")); + assert!(lines.contains("2 files contained matches")); + assert!(lines.contains("2 files searched")); + assert!(lines.contains("seconds")); +}); + +rgtest!( + f411_single_threaded_quiet_search_stats, + |dir: Dir, mut cmd: TestCommand| { + dir.create("sherlock", SHERLOCK); + + let lines = cmd + .arg("--quiet") + .arg("-j1") + .arg("--stats") + .arg("Sherlock") + .stdout(); + assert!(!lines.contains("Sherlock")); + assert!(lines.contains("2 matched lines")); + assert!(lines.contains("1 files contained matches")); + assert!(lines.contains("1 files searched")); + assert!(lines.contains("seconds")); + } +); + +rgtest!(f411_parallel_quiet_search_stats, |dir: Dir, mut cmd: TestCommand| { + dir.create("sherlock_1", SHERLOCK); + dir.create("sherlock_2", SHERLOCK); + + let lines = + cmd.arg("-j2").arg("--quiet").arg("--stats").arg("Sherlock").stdout(); + assert!(!lines.contains("Sherlock")); + assert!(lines.contains("4 matched lines")); + assert!(lines.contains("2 files contained matches")); + assert!(lines.contains("2 files searched")); + assert!(lines.contains("seconds")); +}); + +// See: https://github.com/BurntSushi/ripgrep/issues/416 +rgtest!(f416_crlf, |dir: Dir, mut cmd: TestCommand| { + dir.create("sherlock", SHERLOCK_CRLF); + cmd.arg("--crlf").arg(r"Sherlock$").arg("sherlock"); + + let expected = "\ +For the Doctor Watsons of this world, as opposed to the Sherlock\r +"; + eqnice!(expected, cmd.stdout()); +}); + +// See: https://github.com/BurntSushi/ripgrep/issues/416 +rgtest!(f416_crlf_multiline, |dir: Dir, mut cmd: TestCommand| { + dir.create("sherlock", SHERLOCK_CRLF); + cmd.arg("--crlf").arg("-U").arg(r"Sherlock$").arg("sherlock"); + + let expected = "\ +For the Doctor Watsons of this world, as opposed to the Sherlock\r +"; + eqnice!(expected, cmd.stdout()); +}); + +// See: https://github.com/BurntSushi/ripgrep/issues/416 +rgtest!(f416_crlf_only_matching, |dir: Dir, mut cmd: TestCommand| { + dir.create("sherlock", SHERLOCK_CRLF); + cmd.arg("--crlf").arg("-o").arg(r"Sherlock$").arg("sherlock"); + + let expected = "\ +Sherlock\r +"; + eqnice!(expected, cmd.stdout()); +}); + +// See: https://github.com/BurntSushi/ripgrep/issues/419 +rgtest!(f419_zero_as_shortcut_for_null, |dir: Dir, mut cmd: TestCommand| { + dir.create("sherlock", SHERLOCK); + + cmd.arg("-0").arg("--count").arg("Sherlock"); + eqnice!("sherlock\x002\n", cmd.stdout()); +}); + +rgtest!(f740_passthru, |dir: Dir, mut cmd: TestCommand| { + dir.create("file", "\nfoo\nbar\nfoobar\n\nbaz\n"); + dir.create("patterns", "foo\nbar\n"); + + // We can't assume that the way colour specs are translated to ANSI + // sequences will remain stable, and --replace doesn't currently work with + // pass-through, so for now we don't actually test the match sub-strings + let common_args = &["-n", "--passthru"]; + let foo_expected = "\ +1- +2:foo +3-bar +4:foobar +5- +6-baz +"; + + // With single pattern + cmd.args(common_args).arg("foo").arg("file"); + eqnice!(foo_expected, cmd.stdout()); + + let foo_bar_expected = "\ +1- +2:foo +3:bar +4:foobar +5- +6-baz +"; + + // With multiple -e patterns + let mut cmd = dir.command(); + cmd.args(common_args); + cmd.args(&["-e", "foo", "-e", "bar", "file"]); + eqnice!(foo_bar_expected, cmd.stdout()); + + // With multiple -f patterns + let mut cmd = dir.command(); + cmd.args(common_args); + cmd.args(&["-f", "patterns", "file"]); + eqnice!(foo_bar_expected, cmd.stdout()); + + // -c should override + let mut cmd = dir.command(); + cmd.args(common_args); + cmd.args(&["-c", "foo", "file"]); + eqnice!("2\n", cmd.stdout()); + + let only_foo_expected = "\ +1- +2:foo +3-bar +4:foo +5- +6-baz +"; + + // -o should work + let mut cmd = dir.command(); + cmd.args(common_args); + cmd.args(&["-o", "foo", "file"]); + eqnice!(only_foo_expected, cmd.stdout()); + + let replace_foo_expected = "\ +1- +2:wat +3-bar +4:watbar +5- +6-baz +"; + + // -r should work + let mut cmd = dir.command(); + cmd.args(common_args); + cmd.args(&["-r", "wat", "foo", "file"]); + eqnice!(replace_foo_expected, cmd.stdout()); +}); + +// See: https://github.com/BurntSushi/ripgrep/issues/948 +rgtest!(f948_exit_code_match, |dir: Dir, mut cmd: TestCommand| { + dir.create("sherlock", SHERLOCK); + cmd.arg("."); + + cmd.assert_exit_code(0); +}); + +// See: https://github.com/BurntSushi/ripgrep/issues/948 +rgtest!(f948_exit_code_no_match, |dir: Dir, mut cmd: TestCommand| { + dir.create("sherlock", SHERLOCK); + cmd.arg("NADA"); + + cmd.assert_exit_code(1); +}); + +// See: https://github.com/BurntSushi/ripgrep/issues/948 +rgtest!(f948_exit_code_error, |dir: Dir, mut cmd: TestCommand| { + dir.create("sherlock", SHERLOCK); + cmd.arg("*"); + + cmd.assert_exit_code(2); +}); + +// See: https://github.com/BurntSushi/ripgrep/issues/917 +rgtest!(f917_trim, |dir: Dir, mut cmd: TestCommand| { + const SHERLOCK: &'static str = "\ +zzz + For the Doctor Watsons of this world, as opposed to the Sherlock + Holmeses, success in the province of detective work must always +\tbe, to a very large extent, the result of luck. Sherlock Holmes + can extract a clew from a wisp of straw or a flake of cigar ash; +but Doctor Watson has to have it taken out for him and dusted, + and exhibited clearly, with a label attached. +"; + dir.create("sherlock", SHERLOCK); + cmd.args(&["-n", "-B1", "-A2", "--trim", "Holmeses", "sherlock"]); + + let expected = "\ +2-For the Doctor Watsons of this world, as opposed to the Sherlock +3:Holmeses, success in the province of detective work must always +4-be, to a very large extent, the result of luck. Sherlock Holmes +5-can extract a clew from a wisp of straw or a flake of cigar ash; +"; + eqnice!(expected, cmd.stdout()); +}); + +// See: https://github.com/BurntSushi/ripgrep/issues/917 +// +// This is like f917_trim, except this tests that trimming occurs even when the +// whitespace is part of a match. +rgtest!(f917_trim_match, |dir: Dir, mut cmd: TestCommand| { + const SHERLOCK: &'static str = "\ +zzz + For the Doctor Watsons of this world, as opposed to the Sherlock + Holmeses, success in the province of detective work must always +\tbe, to a very large extent, the result of luck. Sherlock Holmes + can extract a clew from a wisp of straw or a flake of cigar ash; +but Doctor Watson has to have it taken out for him and dusted, + and exhibited clearly, with a label attached. +"; + dir.create("sherlock", SHERLOCK); + cmd.args(&["-n", "-B1", "-A2", "--trim", r"\s+Holmeses", "sherlock"]); + + let expected = "\ +2-For the Doctor Watsons of this world, as opposed to the Sherlock +3:Holmeses, success in the province of detective work must always +4-be, to a very large extent, the result of luck. Sherlock Holmes +5-can extract a clew from a wisp of straw or a flake of cigar ash; +"; + eqnice!(expected, cmd.stdout()); +}); + +rgtest!(f917_trim_multi_standard, |dir: Dir, mut cmd: TestCommand| { + const HAYSTACK: &str = " 0123456789abcdefghijklmnopqrstuvwxyz"; + dir.create("haystack", HAYSTACK); + cmd.args(&["--multiline", "--trim", "-r$0", "--no-filename", r"a\n?bc"]); + + let expected = "0123456789abcdefghijklmnopqrstuvwxyz\n"; + eqnice!(expected, cmd.stdout()); +}); + +rgtest!(f917_trim_max_columns_normal, |dir: Dir, mut cmd: TestCommand| { + const HAYSTACK: &str = " 0123456789abcdefghijklmnopqrstuvwxyz"; + dir.create("haystack", HAYSTACK); + cmd.args(&[ + "--trim", + "--max-columns-preview", + "-M8", + "--no-filename", + "abc", + ]); + + let expected = "01234567 [... omitted end of long line]\n"; + eqnice!(expected, cmd.stdout()); +}); + +rgtest!(f917_trim_max_columns_matches, |dir: Dir, mut cmd: TestCommand| { + const HAYSTACK: &str = " 0123456789abcdefghijklmnopqrstuvwxyz"; + dir.create("haystack", HAYSTACK); + cmd.args(&[ + "--trim", + "--max-columns-preview", + "-M8", + "--color=always", + "--colors=path:none", + "--no-filename", + "abc", + ]); + + let expected = "01234567 [... 1 more match]\n"; + eqnice!(expected, cmd.stdout()); +}); + +rgtest!( + f917_trim_max_columns_multi_standard, + |dir: Dir, mut cmd: TestCommand| { + const HAYSTACK: &str = " 0123456789abcdefghijklmnopqrstuvwxyz"; + dir.create("haystack", HAYSTACK); + cmd.args(&[ + "--multiline", + "--trim", + "--max-columns-preview", + "-M8", + // Force the "slow" printing path without actually + // putting colors in the output. + "--color=always", + "--colors=path:none", + "--no-filename", + r"a\n?bc", + ]); + + let expected = "01234567 [... 1 more match]\n"; + eqnice!(expected, cmd.stdout()); + } +); + +rgtest!( + f917_trim_max_columns_multi_only_matching, + |dir: Dir, mut cmd: TestCommand| { + const HAYSTACK: &str = " 0123456789abcdefghijklmnopqrstuvwxyz"; + dir.create("haystack", HAYSTACK); + cmd.args(&[ + "--multiline", + "--trim", + "--max-columns-preview", + "-M8", + "--only-matching", + "--no-filename", + r".*a\n?bc.*", + ]); + + let expected = "01234567 [... 0 more matches]\n"; + eqnice!(expected, cmd.stdout()); + } +); + +rgtest!( + f917_trim_max_columns_multi_per_match, + |dir: Dir, mut cmd: TestCommand| { + const HAYSTACK: &str = " 0123456789abcdefghijklmnopqrstuvwxyz"; + dir.create("haystack", HAYSTACK); + cmd.args(&[ + "--multiline", + "--trim", + "--max-columns-preview", + "-M8", + "--vimgrep", + "--no-filename", + r".*a\n?bc.*", + ]); + + let expected = "1:1:01234567 [... 0 more matches]\n"; + eqnice!(expected, cmd.stdout()); + } +); + +// See: https://github.com/BurntSushi/ripgrep/issues/993 +rgtest!(f993_null_data, |dir: Dir, mut cmd: TestCommand| { + dir.create("test", "foo\x00bar\x00\x00\x00baz\x00"); + cmd.arg("--null-data").arg(r".+").arg("test"); + + // If we just used -a instead of --null-data, then the result would include + // all NUL bytes. + let expected = "foo\x00bar\x00baz\x00"; + eqnice!(expected, cmd.stdout()); +}); + +// See: https://github.com/BurntSushi/ripgrep/issues/1078 +// +// N.B. There are many more tests in the grep-printer crate. +rgtest!(f1078_max_columns_preview1, |dir: Dir, mut cmd: TestCommand| { + dir.create("sherlock", SHERLOCK); + cmd.args(&[ + "-M46", + "--max-columns-preview", + "exhibited|dusted|has to have it", + ]); + + let expected = "\ +sherlock:but Doctor Watson has to have it taken out for [... omitted end of long line] +sherlock:and exhibited clearly, with a label attached. +"; + eqnice!(expected, cmd.stdout()); +}); + +rgtest!(f1078_max_columns_preview2, |dir: Dir, mut cmd: TestCommand| { + dir.create("sherlock", SHERLOCK); + cmd.args(&[ + "-M43", + "--max-columns-preview", + // Doing a replacement forces ripgrep to show the number of remaining + // matches. Normally, this happens by default when printing a tty with + // colors. + "-rxxx", + "exhibited|dusted|has to have it", + ]); + + let expected = "\ +sherlock:but Doctor Watson xxx taken out for him and [... 1 more match] +sherlock:and xxx clearly, with a label attached. +"; + eqnice!(expected, cmd.stdout()); +}); + +// See: https://github.com/BurntSushi/ripgrep/issues/1138 +rgtest!(f1138_no_ignore_dot, |dir: Dir, mut cmd: TestCommand| { + dir.create_dir(".git"); + dir.create(".gitignore", "foo"); + dir.create(".ignore", "bar"); + dir.create(".fzf-ignore", "quux"); + dir.create("foo", ""); + dir.create("bar", ""); + dir.create("quux", ""); + + cmd.arg("--sort").arg("path").arg("--files"); + eqnice!("quux\n", cmd.stdout()); + eqnice!("bar\nquux\n", cmd.arg("--no-ignore-dot").stdout()); + eqnice!("bar\n", cmd.arg("--ignore-file").arg(".fzf-ignore").stdout()); +}); + +// See: https://github.com/BurntSushi/ripgrep/issues/1155 +rgtest!(f1155_auto_hybrid_regex, |dir: Dir, mut cmd: TestCommand| { + // No sense in testing a hybrid regex engine with only one engine! + if !dir.is_pcre2() { + return; + } + + dir.create("sherlock", SHERLOCK); + cmd.arg("--no-pcre2").arg("--auto-hybrid-regex").arg(r"(?<=the )Sherlock"); + + let expected = "\ +sherlock:For the Doctor Watsons of this world, as opposed to the Sherlock +"; + eqnice!(expected, cmd.stdout()); +}); + +// See: https://github.com/BurntSushi/ripgrep/issues/1207 +// +// Tests if without encoding 'none' flag null bytes are consumed by automatic +// encoding detection. +rgtest!(f1207_auto_encoding, |dir: Dir, mut cmd: TestCommand| { + dir.create_bytes("foo", b"\xFF\xFE\x00\x62"); + cmd.arg("-a").arg("\\x00").arg("foo"); + cmd.assert_exit_code(1); +}); + +// See: https://github.com/BurntSushi/ripgrep/issues/1207 +// +// Tests if encoding 'none' flag does treat file as raw bytes +rgtest!(f1207_ignore_encoding, |dir: Dir, mut cmd: TestCommand| { + // PCRE2 chokes on this test because it can't search invalid non-UTF-8 + // and the point of this test is to search raw UTF-16. + if dir.is_pcre2() { + return; + } + + dir.create_bytes("foo", b"\xFF\xFE\x00\x62"); + cmd.arg("--encoding").arg("none").arg("-a").arg("\\x00").arg("foo"); + eqnice!("\u{FFFD}\u{FFFD}\x00b\n", cmd.stdout()); +}); + +// See: https://github.com/BurntSushi/ripgrep/issues/1414 +rgtest!(f1414_no_require_git, |dir: Dir, mut cmd: TestCommand| { + dir.create(".gitignore", "foo"); + dir.create("foo", ""); + dir.create("bar", ""); + + let stdout = cmd.args(&["--sort", "path", "--files"]).stdout(); + eqnice!("bar\nfoo\n", stdout); + + let stdout = + cmd.args(&["--sort", "path", "--files", "--no-require-git"]).stdout(); + eqnice!("bar\n", stdout); + + let stdout = cmd + .args(&[ + "--sort", + "path", + "--files", + "--no-require-git", + "--require-git", + ]) + .stdout(); + eqnice!("bar\nfoo\n", stdout); +}); + +// See: https://github.com/BurntSushi/ripgrep/pull/1420 +rgtest!(f1420_no_ignore_exclude, |dir: Dir, mut cmd: TestCommand| { + dir.create_dir(".git/info"); + dir.create(".git/info/exclude", "foo"); + dir.create("bar", ""); + dir.create("foo", ""); + + cmd.arg("--sort").arg("path").arg("--files"); + eqnice!("bar\n", cmd.stdout()); + eqnice!("bar\nfoo\n", cmd.arg("--no-ignore-exclude").stdout()); +}); + +// See: https://github.com/BurntSushi/ripgrep/pull/1466 +rgtest!(f1466_no_ignore_files, |dir: Dir, mut cmd: TestCommand| { + dir.create(".myignore", "bar"); + dir.create("bar", ""); + dir.create("foo", ""); + + // Test that --no-ignore-files disables --ignore-file. + // And that --ignore-files overrides --no-ignore-files. + cmd.arg("--sort").arg("path").arg("--files"); + eqnice!("bar\nfoo\n", cmd.stdout()); + eqnice!("foo\n", cmd.arg("--ignore-file").arg(".myignore").stdout()); + eqnice!("bar\nfoo\n", cmd.arg("--no-ignore-files").stdout()); + eqnice!("foo\n", cmd.arg("--ignore-files").stdout()); + + // Test that the -u flag does not disable --ignore-file. + let mut cmd = dir.command(); + cmd.arg("--sort").arg("path").arg("--files"); + cmd.arg("--ignore-file").arg(".myignore"); + eqnice!("foo\n", cmd.stdout()); + eqnice!("foo\n", cmd.arg("-u").stdout()); +}); + +// See: https://github.com/BurntSushi/ripgrep/pull/2361 +rgtest!(f2361_sort_nested_files, |dir: Dir, mut cmd: TestCommand| { + use std::{thread::sleep, time::Duration}; + + if crate::util::is_cross() { + return; + } + dir.create("foo", "1"); + sleep(Duration::from_millis(200)); + dir.create_dir("dir"); + sleep(Duration::from_millis(200)); + dir.create(dir.path().join("dir").join("bar"), "1"); + + cmd.arg("--sort").arg("accessed").arg("--files"); + eqnice!("foo\ndir/bar\n", cmd.stdout()); + + dir.create("foo", "2"); + sleep(Duration::from_millis(200)); + dir.create(dir.path().join("dir").join("bar"), "2"); + sleep(Duration::from_millis(200)); + + cmd.arg("--sort").arg("accessed").arg("--files"); + eqnice!("foo\ndir/bar\n", cmd.stdout()); +}); + +// See: https://github.com/BurntSushi/ripgrep/issues/1404 +rgtest!(f1404_nothing_searched_warning, |dir: Dir, mut cmd: TestCommand| { + dir.create(".ignore", "ignored-dir/**"); + dir.create_dir("ignored-dir"); + dir.create("ignored-dir/foo", "needle"); + + // Test that, if ripgrep searches only ignored folders/files, then there + // is a non-zero exit code. + cmd.arg("needle"); + cmd.assert_err(); + + // Test that we actually get an error message that we expect. + let output = cmd.raw_output(); + let stderr = String::from_utf8_lossy(&output.stderr); + let expected = "\ + rg: No files were searched, which means ripgrep probably applied \ + a filter you didn't expect.\n\ + Running with --debug will show why files are being skipped.\n\ + "; + eqnice!(expected, stderr); +}); + +// See: https://github.com/BurntSushi/ripgrep/issues/1404 +rgtest!(f1404_nothing_searched_ignored, |dir: Dir, mut cmd: TestCommand| { + dir.create(".ignore", "ignored-dir/**"); + dir.create_dir("ignored-dir"); + dir.create("ignored-dir/foo", "needle"); + + // Test that, if ripgrep searches only ignored folders/files, then there + // is a non-zero exit code. + cmd.arg("--no-messages").arg("needle"); + cmd.assert_err(); + + // But since --no-messages is given, there should not be any error message + // printed. + let output = cmd.raw_output(); + let stderr = String::from_utf8_lossy(&output.stderr); + let expected = ""; + eqnice!(expected, stderr); +}); + +// See: https://github.com/BurntSushi/ripgrep/issues/1842 +rgtest!(f1842_field_context_separator, |dir: Dir, _: TestCommand| { + dir.create("sherlock", SHERLOCK); + + // Test the default. + let base = &["-n", "-A1", "Doctor Watsons", "sherlock"]; + let expected = "\ +1:For the Doctor Watsons of this world, as opposed to the Sherlock +2-Holmeses, success in the province of detective work must always +"; + eqnice!(expected, dir.command().args(base).stdout()); + + // Test that it can be overridden. + let mut args = vec!["--field-context-separator", "!"]; + args.extend(base); + let expected = "\ +1:For the Doctor Watsons of this world, as opposed to the Sherlock +2!Holmeses, success in the province of detective work must always +"; + eqnice!(expected, dir.command().args(&args).stdout()); + + // Test that it can use multiple bytes. + let mut args = vec!["--field-context-separator", "!!"]; + args.extend(base); + let expected = "\ +1:For the Doctor Watsons of this world, as opposed to the Sherlock +2!!Holmeses, success in the province of detective work must always +"; + eqnice!(expected, dir.command().args(&args).stdout()); + + // Test that unescaping works. + let mut args = vec!["--field-context-separator", r"\x7F"]; + args.extend(base); + let expected = "\ +1:For the Doctor Watsons of this world, as opposed to the Sherlock +2\x7FHolmeses, success in the province of detective work must always +"; + eqnice!(expected, dir.command().args(&args).stdout()); + + // Test that an empty separator is OK. + let mut args = vec!["--field-context-separator", r""]; + args.extend(base); + let expected = "\ +1:For the Doctor Watsons of this world, as opposed to the Sherlock +2Holmeses, success in the province of detective work must always +"; + eqnice!(expected, dir.command().args(&args).stdout()); +}); + +// See: https://github.com/BurntSushi/ripgrep/issues/1842 +rgtest!(f1842_field_match_separator, |dir: Dir, _: TestCommand| { + dir.create("sherlock", SHERLOCK); + + // Test the default. + let base = &["-n", "Doctor Watsons", "sherlock"]; + let expected = "\ +1:For the Doctor Watsons of this world, as opposed to the Sherlock +"; + eqnice!(expected, dir.command().args(base).stdout()); + + // Test that it can be overridden. + let mut args = vec!["--field-match-separator", "!"]; + args.extend(base); + let expected = "\ +1!For the Doctor Watsons of this world, as opposed to the Sherlock +"; + eqnice!(expected, dir.command().args(&args).stdout()); + + // Test that it can use multiple bytes. + let mut args = vec!["--field-match-separator", "!!"]; + args.extend(base); + let expected = "\ +1!!For the Doctor Watsons of this world, as opposed to the Sherlock +"; + eqnice!(expected, dir.command().args(&args).stdout()); + + // Test that unescaping works. + let mut args = vec!["--field-match-separator", r"\x7F"]; + args.extend(base); + let expected = "\ +1\x7FFor the Doctor Watsons of this world, as opposed to the Sherlock +"; + eqnice!(expected, dir.command().args(&args).stdout()); + + // Test that an empty separator is OK. + let mut args = vec!["--field-match-separator", r""]; + args.extend(base); + let expected = "\ +1For the Doctor Watsons of this world, as opposed to the Sherlock +"; + eqnice!(expected, dir.command().args(&args).stdout()); +}); + +// See: https://github.com/BurntSushi/ripgrep/issues/2288 +rgtest!(f2288_context_partial_override, |dir: Dir, mut cmd: TestCommand| { + dir.create("test", "1\n2\n3\n4\n5\n6\n7\n8\n9\n"); + cmd.args(&["-C1", "-A2", "5", "test"]); + eqnice!("4\n5\n6\n7\n", cmd.stdout()); +}); + +// See: https://github.com/BurntSushi/ripgrep/issues/2288 +rgtest!( + f2288_context_partial_override_rev, + |dir: Dir, mut cmd: TestCommand| { + dir.create("test", "1\n2\n3\n4\n5\n6\n7\n8\n9\n"); + cmd.args(&["-A2", "-C1", "5", "test"]); + eqnice!("4\n5\n6\n7\n", cmd.stdout()); + } +); + +rgtest!(no_context_sep, |dir: Dir, mut cmd: TestCommand| { + dir.create("test", "foo\nctx\nbar\nctx\nfoo\nctx"); + cmd.args(&["-A1", "--no-context-separator", "foo", "test"]); + eqnice!("foo\nctx\nfoo\nctx\n", cmd.stdout()); +}); + +rgtest!(no_context_sep_overrides, |dir: Dir, mut cmd: TestCommand| { + dir.create("test", "foo\nctx\nbar\nctx\nfoo\nctx"); + cmd.args(&[ + "-A1", + "--context-separator", + "AAA", + "--no-context-separator", + "foo", + "test", + ]); + eqnice!("foo\nctx\nfoo\nctx\n", cmd.stdout()); +}); + +rgtest!(no_context_sep_overridden, |dir: Dir, mut cmd: TestCommand| { + dir.create("test", "foo\nctx\nbar\nctx\nfoo\nctx"); + cmd.args(&[ + "-A1", + "--no-context-separator", + "--context-separator", + "AAA", + "foo", + "test", + ]); + eqnice!("foo\nctx\nAAA\nfoo\nctx\n", cmd.stdout()); +}); + +rgtest!(context_sep, |dir: Dir, mut cmd: TestCommand| { + dir.create("test", "foo\nctx\nbar\nctx\nfoo\nctx"); + cmd.args(&["-A1", "--context-separator", "AAA", "foo", "test"]); + eqnice!("foo\nctx\nAAA\nfoo\nctx\n", cmd.stdout()); +}); + +rgtest!(context_sep_default, |dir: Dir, mut cmd: TestCommand| { + dir.create("test", "foo\nctx\nbar\nctx\nfoo\nctx"); + cmd.args(&["-A1", "foo", "test"]); + eqnice!("foo\nctx\n--\nfoo\nctx\n", cmd.stdout()); +}); + +rgtest!(context_sep_empty, |dir: Dir, mut cmd: TestCommand| { + dir.create("test", "foo\nctx\nbar\nctx\nfoo\nctx"); + cmd.args(&["-A1", "--context-separator", "", "foo", "test"]); + eqnice!("foo\nctx\n\nfoo\nctx\n", cmd.stdout()); +}); + +rgtest!(no_unicode, |dir: Dir, mut cmd: TestCommand| { + dir.create("test", "δ"); + cmd.arg("-i").arg("--no-unicode").arg("Δ").assert_err(); +}); + +// See: https://github.com/BurntSushi/ripgrep/issues/1790 +rgtest!(stop_on_nonmatch, |dir: Dir, mut cmd: TestCommand| { + dir.create("test", "line1\nline2\nline3\nline4\nline5"); + cmd.args(&["--stop-on-nonmatch", "[235]"]); + eqnice!("test:line2\ntest:line3\n", cmd.stdout()); +}); diff --git a/userland/ripgrep/tests/hay.rs b/userland/ripgrep/tests/hay.rs new file mode 100644 index 0000000..15a53b1 --- /dev/null +++ b/userland/ripgrep/tests/hay.rs @@ -0,0 +1,17 @@ +pub const SHERLOCK: &'static str = "\ +For the Doctor Watsons of this world, as opposed to the Sherlock +Holmeses, success in the province of detective work must always +be, to a very large extent, the result of luck. Sherlock Holmes +can extract a clew from a wisp of straw or a flake of cigar ash; +but Doctor Watson has to have it taken out for him and dusted, +and exhibited clearly, with a label attached. +"; + +pub const SHERLOCK_CRLF: &'static str = "\ +For the Doctor Watsons of this world, as opposed to the Sherlock\r +Holmeses, success in the province of detective work must always\r +be, to a very large extent, the result of luck. Sherlock Holmes\r +can extract a clew from a wisp of straw or a flake of cigar ash;\r +but Doctor Watson has to have it taken out for him and dusted,\r +and exhibited clearly, with a label attached.\r +"; diff --git a/userland/ripgrep/tests/json.rs b/userland/ripgrep/tests/json.rs new file mode 100644 index 0000000..d1233d7 --- /dev/null +++ b/userland/ripgrep/tests/json.rs @@ -0,0 +1,441 @@ +use std::time; + +use serde_derive::Deserialize; +use serde_json as json; + +use crate::hay::{SHERLOCK, SHERLOCK_CRLF}; +use crate::util::{Dir, TestCommand}; + +#[derive(Clone, Debug, Deserialize, PartialEq, Eq)] +#[serde(tag = "type", content = "data")] +#[serde(rename_all = "snake_case")] +enum Message { + Begin(Begin), + End(End), + Match(Match), + Context(Context), + Summary(Summary), +} + +impl Message { + fn unwrap_begin(&self) -> Begin { + match *self { + Message::Begin(ref x) => x.clone(), + ref x => panic!("expected Message::Begin but got {x:?}"), + } + } + + fn unwrap_end(&self) -> End { + match *self { + Message::End(ref x) => x.clone(), + ref x => panic!("expected Message::End but got {x:?}"), + } + } + + fn unwrap_match(&self) -> Match { + match *self { + Message::Match(ref x) => x.clone(), + ref x => panic!("expected Message::Match but got {x:?}"), + } + } + + fn unwrap_context(&self) -> Context { + match *self { + Message::Context(ref x) => x.clone(), + ref x => panic!("expected Message::Context but got {x:?}"), + } + } + + fn unwrap_summary(&self) -> Summary { + match *self { + Message::Summary(ref x) => x.clone(), + ref x => panic!("expected Message::Summary but got {x:?}"), + } + } +} + +#[derive(Clone, Debug, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +struct Begin { + path: Option, +} + +#[derive(Clone, Debug, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +struct End { + path: Option, + binary_offset: Option, + stats: Stats, +} + +#[derive(Clone, Debug, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +struct Summary { + elapsed_total: Duration, + stats: Stats, +} + +#[derive(Clone, Debug, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +struct Match { + path: Option, + lines: Data, + line_number: Option, + absolute_offset: u64, + submatches: Vec, +} + +#[derive(Clone, Debug, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +struct Context { + path: Option, + lines: Data, + line_number: Option, + absolute_offset: u64, + submatches: Vec, +} + +#[derive(Clone, Debug, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +struct SubMatch { + #[serde(rename = "match")] + m: Data, + replacement: Option, + start: usize, + end: usize, +} + +#[derive(Clone, Debug, Deserialize, PartialEq, Eq)] +#[serde(untagged)] +enum Data { + Text { text: String }, + // This variant is used when the data isn't valid UTF-8. The bytes are + // base64 encoded, so using a String here is OK. + Bytes { bytes: String }, +} + +impl Data { + fn text(s: &str) -> Data { + Data::Text { text: s.to_string() } + } + fn bytes(s: &str) -> Data { + Data::Bytes { bytes: s.to_string() } + } +} + +#[derive(Clone, Debug, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +struct Stats { + elapsed: Duration, + searches: u64, + searches_with_match: u64, + bytes_searched: u64, + bytes_printed: u64, + matched_lines: u64, + matches: u64, +} + +#[derive(Clone, Debug, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +struct Duration { + #[serde(flatten)] + duration: time::Duration, + human: String, +} + +/// Decode JSON Lines into a Vec. If there was an error decoding, +/// this function panics. +fn json_decode(jsonlines: &str) -> Vec { + json::Deserializer::from_str(jsonlines) + .into_iter() + .collect::, _>>() + .unwrap() +} + +rgtest!(basic, |dir: Dir, mut cmd: TestCommand| { + dir.create("sherlock", SHERLOCK); + cmd.arg("--json").arg("-B1").arg("Sherlock Holmes").arg("sherlock"); + + let msgs = json_decode(&cmd.stdout()); + + assert_eq!( + msgs[0].unwrap_begin(), + Begin { path: Some(Data::text("sherlock")) } + ); + assert_eq!( + msgs[1].unwrap_context(), + Context { + path: Some(Data::text("sherlock")), + lines: Data::text( + "Holmeses, success in the province of \ + detective work must always\n", + ), + line_number: Some(2), + absolute_offset: 65, + submatches: vec![], + } + ); + assert_eq!( + msgs[2].unwrap_match(), + Match { + path: Some(Data::text("sherlock")), + lines: Data::text( + "be, to a very large extent, the result of luck. \ + Sherlock Holmes\n", + ), + line_number: Some(3), + absolute_offset: 129, + submatches: vec![SubMatch { + m: Data::text("Sherlock Holmes"), + replacement: None, + start: 48, + end: 63, + },], + } + ); + assert_eq!(msgs[3].unwrap_end().path, Some(Data::text("sherlock"))); + assert_eq!(msgs[3].unwrap_end().binary_offset, None); + assert_eq!(msgs[4].unwrap_summary().stats.searches_with_match, 1); + assert_eq!(msgs[4].unwrap_summary().stats.bytes_printed, 494); +}); + +rgtest!(replacement, |dir: Dir, mut cmd: TestCommand| { + dir.create("sherlock", SHERLOCK); + cmd.arg("--json") + .arg("-B1") + .arg("Sherlock Holmes") + .args(["-r", "John Watson"]) + .arg("sherlock"); + + let msgs = json_decode(&cmd.stdout()); + + assert_eq!( + msgs[0].unwrap_begin(), + Begin { path: Some(Data::text("sherlock")) } + ); + assert_eq!( + msgs[1].unwrap_context(), + Context { + path: Some(Data::text("sherlock")), + lines: Data::text( + "Holmeses, success in the province of \ + detective work must always\n", + ), + line_number: Some(2), + absolute_offset: 65, + submatches: vec![], + } + ); + assert_eq!( + msgs[2].unwrap_match(), + Match { + path: Some(Data::text("sherlock")), + lines: Data::text( + "be, to a very large extent, the result of luck. \ + Sherlock Holmes\n", + ), + line_number: Some(3), + absolute_offset: 129, + submatches: vec![SubMatch { + m: Data::text("Sherlock Holmes"), + replacement: Some(Data::text("John Watson")), + start: 48, + end: 63, + },], + } + ); + assert_eq!(msgs[3].unwrap_end().path, Some(Data::text("sherlock"))); + assert_eq!(msgs[3].unwrap_end().binary_offset, None); + assert_eq!(msgs[4].unwrap_summary().stats.searches_with_match, 1); + assert_eq!(msgs[4].unwrap_summary().stats.bytes_printed, 531); +}); + +rgtest!(quiet_stats, |dir: Dir, mut cmd: TestCommand| { + dir.create("sherlock", SHERLOCK); + cmd.arg("--json") + .arg("--quiet") + .arg("--stats") + .arg("Sherlock Holmes") + .arg("sherlock"); + + let msgs = json_decode(&cmd.stdout()); + assert_eq!(msgs[0].unwrap_summary().stats.searches_with_match, 1); + assert_eq!(msgs[0].unwrap_summary().stats.bytes_searched, 367); +}); + +#[cfg(unix)] +rgtest!(notutf8, |dir: Dir, mut cmd: TestCommand| { + use std::ffi::OsStr; + use std::os::unix::ffi::OsStrExt; + + // This test does not work with PCRE2 because PCRE2 does not support the + // `u` flag. + if dir.is_pcre2() { + return; + } + // macOS doesn't like this either... sigh. + if cfg!(target_os = "macos") { + return; + } + + let name = &b"foo\xFFbar"[..]; + let contents = &b"quux\xFFbaz"[..]; + + // APFS does not support creating files with invalid UTF-8 bytes, so just + // skip the test if we can't create our file. Presumably we don't need this + // check if we're already skipping it on macOS, but maybe other file + // systems won't like this test either? + if !dir.try_create_bytes(OsStr::from_bytes(name), contents).is_ok() { + return; + } + cmd.arg("--json").arg(r"(?-u)\xFF"); + + let msgs = json_decode(&cmd.stdout()); + + assert_eq!( + msgs[0].unwrap_begin(), + Begin { path: Some(Data::bytes("Zm9v/2Jhcg==")) } + ); + assert_eq!( + msgs[1].unwrap_match(), + Match { + path: Some(Data::bytes("Zm9v/2Jhcg==")), + lines: Data::bytes("cXV1eP9iYXo="), + line_number: Some(1), + absolute_offset: 0, + submatches: vec![SubMatch { + m: Data::bytes("/w=="), + replacement: None, + start: 4, + end: 5, + },], + } + ); +}); + +rgtest!(notutf8_file, |dir: Dir, mut cmd: TestCommand| { + use std::ffi::OsStr; + + // This test does not work with PCRE2 because PCRE2 does not support the + // `u` flag. + if dir.is_pcre2() { + return; + } + + let name = "foo"; + let contents = &b"quux\xFFbaz"[..]; + + // APFS does not support creating files with invalid UTF-8 bytes, so just + // skip the test if we can't create our file. + if !dir.try_create_bytes(OsStr::new(name), contents).is_ok() { + return; + } + cmd.arg("--json").arg(r"(?-u)\xFF"); + + let msgs = json_decode(&cmd.stdout()); + + assert_eq!( + msgs[0].unwrap_begin(), + Begin { path: Some(Data::text("foo")) } + ); + assert_eq!( + msgs[1].unwrap_match(), + Match { + path: Some(Data::text("foo")), + lines: Data::bytes("cXV1eP9iYXo="), + line_number: Some(1), + absolute_offset: 0, + submatches: vec![SubMatch { + m: Data::bytes("/w=="), + replacement: None, + start: 4, + end: 5, + },], + } + ); +}); + +// See: https://github.com/BurntSushi/ripgrep/issues/416 +// +// This test in particular checks that our match does _not_ include the `\r` +// even though the '$' may be rewritten as '(?:\r??$)' and could thus include +// `\r` in the match. +rgtest!(crlf, |dir: Dir, mut cmd: TestCommand| { + dir.create("sherlock", SHERLOCK_CRLF); + cmd.arg("--json").arg("--crlf").arg(r"Sherlock$").arg("sherlock"); + + let msgs = json_decode(&cmd.stdout()); + + assert_eq!( + msgs[1].unwrap_match().submatches[0].clone(), + SubMatch { + m: Data::text("Sherlock"), + replacement: None, + start: 56, + end: 64 + }, + ); +}); + +// See: https://github.com/BurntSushi/ripgrep/issues/1095 +// +// This test checks that we don't drop the \r\n in a matching line when --crlf +// mode is enabled. +rgtest!(r1095_missing_crlf, |dir: Dir, mut cmd: TestCommand| { + dir.create("foo", "test\r\n"); + + // Check without --crlf flag. + let msgs = json_decode(&cmd.arg("--json").arg("test").stdout()); + assert_eq!(msgs.len(), 4); + assert_eq!(msgs[1].unwrap_match().lines, Data::text("test\r\n")); + + // Now check with --crlf flag. + let msgs = json_decode(&cmd.arg("--crlf").stdout()); + assert_eq!(msgs.len(), 4); + assert_eq!(msgs[1].unwrap_match().lines, Data::text("test\r\n")); +}); + +// See: https://github.com/BurntSushi/ripgrep/issues/1095 +// +// This test checks that we don't return empty submatches when matching a `\n` +// in CRLF mode. +rgtest!(r1095_crlf_empty_match, |dir: Dir, mut cmd: TestCommand| { + dir.create("foo", "test\r\n\n"); + + // Check without --crlf flag. + let msgs = json_decode(&cmd.arg("-U").arg("--json").arg("\n").stdout()); + assert_eq!(msgs.len(), 4); + + let m = msgs[1].unwrap_match(); + assert_eq!(m.lines, Data::text("test\r\n\n")); + assert_eq!(m.submatches[0].m, Data::text("\n")); + assert_eq!(m.submatches[1].m, Data::text("\n")); + + // Now check with --crlf flag. + let msgs = json_decode(&cmd.arg("--crlf").stdout()); + assert_eq!(msgs.len(), 4); + + let m = msgs[1].unwrap_match(); + assert_eq!(m.lines, Data::text("test\r\n\n")); + assert_eq!(m.submatches[0].m, Data::text("\n")); + assert_eq!(m.submatches[1].m, Data::text("\n")); +}); + +// See: https://github.com/BurntSushi/ripgrep/issues/1412 +rgtest!(r1412_look_behind_match_missing, |dir: Dir, mut cmd: TestCommand| { + // Only PCRE2 supports look-around. + if !dir.is_pcre2() { + return; + } + + dir.create("test", "foo\nbar\n"); + + let msgs = json_decode( + &cmd.arg("-U").arg("--json").arg(r"(?<=foo\n)bar").stdout(), + ); + assert_eq!(msgs.len(), 4); + + let m = msgs[1].unwrap_match(); + assert_eq!(m.lines, Data::text("bar\n")); + assert_eq!(m.submatches.len(), 1); +}); diff --git a/userland/ripgrep/tests/macros.rs b/userland/ripgrep/tests/macros.rs new file mode 100644 index 0000000..7e5958c --- /dev/null +++ b/userland/ripgrep/tests/macros.rs @@ -0,0 +1,61 @@ +#[macro_export] +macro_rules! rgtest { + ($name:ident, $fun:expr) => { + #[test] + fn $name() { + let (dir, cmd) = crate::util::setup(stringify!($name)); + $fun(dir, cmd); + + if cfg!(feature = "pcre2") { + let (dir, cmd) = crate::util::setup_pcre2(stringify!($name)); + $fun(dir, cmd); + } + } + }; +} + +#[macro_export] +macro_rules! eqnice { + ($expected:expr, $got:expr) => { + let expected = &*$expected; + let got = &*$got; + if expected != got { + panic!(" +printed outputs differ! + +expected: +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +{} +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +got: +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +{} +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +", expected, got); + } + } +} + +#[macro_export] +macro_rules! eqnice_repr { + ($expected:expr, $got:expr) => { + let expected = &*$expected; + let got = &*$got; + if expected != got { + panic!(" +printed outputs differ! + +expected: +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +{:?} +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +got: +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +{:?} +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +", expected, got); + } + } +} diff --git a/userland/ripgrep/tests/misc.rs b/userland/ripgrep/tests/misc.rs new file mode 100644 index 0000000..268e464 --- /dev/null +++ b/userland/ripgrep/tests/misc.rs @@ -0,0 +1,1198 @@ +use crate::hay::SHERLOCK; +use crate::util::{Dir, TestCommand, cmd_exists, sort_lines}; + +// This file contains "miscellaneous" tests that were either written before +// features were tracked more explicitly, or were simply written without +// linking them to a specific issue number. We should try to minimize the +// addition of more tests in this file and instead add them to either the +// regression test suite or the feature test suite (found in regression.rs and +// feature.rs, respectively). + +rgtest!(single_file, |dir: Dir, mut cmd: TestCommand| { + dir.create("sherlock", SHERLOCK); + + let expected = "\ +For the Doctor Watsons of this world, as opposed to the Sherlock +be, to a very large extent, the result of luck. Sherlock Holmes +"; + eqnice!(expected, cmd.arg("Sherlock").arg("sherlock").stdout()); +}); + +rgtest!(dir, |dir: Dir, mut cmd: TestCommand| { + dir.create("sherlock", SHERLOCK); + + let expected = "\ +sherlock:For the Doctor Watsons of this world, as opposed to the Sherlock +sherlock:be, to a very large extent, the result of luck. Sherlock Holmes +"; + eqnice!(expected, cmd.arg("Sherlock").stdout()); +}); + +rgtest!(line_numbers, |dir: Dir, mut cmd: TestCommand| { + dir.create("sherlock", SHERLOCK); + + let expected = "\ +1:For the Doctor Watsons of this world, as opposed to the Sherlock +3:be, to a very large extent, the result of luck. Sherlock Holmes +"; + eqnice!(expected, cmd.arg("-n").arg("Sherlock").arg("sherlock").stdout()); +}); + +rgtest!(columns, |dir: Dir, mut cmd: TestCommand| { + dir.create("sherlock", SHERLOCK); + cmd.arg("--column").arg("Sherlock").arg("sherlock"); + + let expected = "\ +1:57:For the Doctor Watsons of this world, as opposed to the Sherlock +3:49:be, to a very large extent, the result of luck. Sherlock Holmes +"; + eqnice!(expected, cmd.stdout()); +}); + +rgtest!(with_filename, |dir: Dir, mut cmd: TestCommand| { + dir.create("sherlock", SHERLOCK); + cmd.arg("-H").arg("Sherlock").arg("sherlock"); + + let expected = "\ +sherlock:For the Doctor Watsons of this world, as opposed to the Sherlock +sherlock:be, to a very large extent, the result of luck. Sherlock Holmes +"; + eqnice!(expected, cmd.stdout()); +}); + +rgtest!(with_heading, |dir: Dir, mut cmd: TestCommand| { + dir.create("sherlock", SHERLOCK); + cmd.args(&[ + // This forces the issue since --with-filename is disabled by default + // when searching one file. + "--with-filename", + "--heading", + "Sherlock", + "sherlock", + ]); + + let expected = "\ +sherlock +For the Doctor Watsons of this world, as opposed to the Sherlock +be, to a very large extent, the result of luck. Sherlock Holmes +"; + eqnice!(expected, cmd.stdout()); +}); + +rgtest!(with_heading_default, |dir: Dir, mut cmd: TestCommand| { + // Search two or more and get --with-filename enabled by default. + // Use -j1 to get deterministic results. + dir.create("sherlock", SHERLOCK); + dir.create("foo", "Sherlock Holmes lives on Baker Street."); + cmd.arg("-j1").arg("--heading").arg("Sherlock"); + + let expected = "\ +foo +Sherlock Holmes lives on Baker Street. + +sherlock +For the Doctor Watsons of this world, as opposed to the Sherlock +be, to a very large extent, the result of luck. Sherlock Holmes +"; + eqnice!(sort_lines(expected), sort_lines(&cmd.stdout())); +}); + +rgtest!(inverted, |dir: Dir, mut cmd: TestCommand| { + dir.create("sherlock", SHERLOCK); + cmd.arg("-v").arg("Sherlock").arg("sherlock"); + + let expected = "\ +Holmeses, success in the province of detective work must always +can extract a clew from a wisp of straw or a flake of cigar ash; +but Doctor Watson has to have it taken out for him and dusted, +and exhibited clearly, with a label attached. +"; + eqnice!(expected, cmd.stdout()); +}); + +rgtest!(inverted_line_numbers, |dir: Dir, mut cmd: TestCommand| { + dir.create("sherlock", SHERLOCK); + cmd.arg("-n").arg("-v").arg("Sherlock").arg("sherlock"); + + let expected = "\ +2:Holmeses, success in the province of detective work must always +4:can extract a clew from a wisp of straw or a flake of cigar ash; +5:but Doctor Watson has to have it taken out for him and dusted, +6:and exhibited clearly, with a label attached. +"; + eqnice!(expected, cmd.stdout()); +}); + +rgtest!(case_insensitive, |dir: Dir, mut cmd: TestCommand| { + dir.create("sherlock", SHERLOCK); + cmd.arg("-i").arg("sherlock").arg("sherlock"); + + let expected = "\ +For the Doctor Watsons of this world, as opposed to the Sherlock +be, to a very large extent, the result of luck. Sherlock Holmes +"; + eqnice!(expected, cmd.stdout()); +}); + +rgtest!(word, |dir: Dir, mut cmd: TestCommand| { + dir.create("sherlock", SHERLOCK); + cmd.arg("-w").arg("as").arg("sherlock"); + + let expected = "\ +For the Doctor Watsons of this world, as opposed to the Sherlock +"; + eqnice!(expected, cmd.stdout()); +}); + +rgtest!(word_period, |dir: Dir, mut cmd: TestCommand| { + dir.create("haystack", "..."); + cmd.arg("-ow").arg(".").arg("haystack"); + + let expected = "\ +. +. +. +"; + eqnice!(expected, cmd.stdout()); +}); + +rgtest!(line, |dir: Dir, mut cmd: TestCommand| { + dir.create("sherlock", SHERLOCK); + cmd.args(&[ + "-x", + "Watson|and exhibited clearly, with a label attached.", + "sherlock", + ]); + + let expected = "\ +and exhibited clearly, with a label attached. +"; + eqnice!(expected, cmd.stdout()); +}); + +rgtest!(literal, |dir: Dir, mut cmd: TestCommand| { + dir.create("sherlock", SHERLOCK); + dir.create("file", "blib\n()\nblab\n"); + cmd.arg("-F").arg("()").arg("file"); + + eqnice!("()\n", cmd.stdout()); +}); + +rgtest!(quiet, |dir: Dir, mut cmd: TestCommand| { + dir.create("sherlock", SHERLOCK); + cmd.arg("-q").arg("Sherlock").arg("sherlock"); + + assert!(cmd.stdout().is_empty()); +}); + +rgtest!(replace, |dir: Dir, mut cmd: TestCommand| { + dir.create("sherlock", SHERLOCK); + cmd.arg("-r").arg("FooBar").arg("Sherlock").arg("sherlock"); + + let expected = "\ +For the Doctor Watsons of this world, as opposed to the FooBar +be, to a very large extent, the result of luck. FooBar Holmes +"; + eqnice!(expected, cmd.stdout()); +}); + +rgtest!(replace_groups, |dir: Dir, mut cmd: TestCommand| { + dir.create("sherlock", SHERLOCK); + cmd.args(&["-r", "$2, $1", "([A-Z][a-z]+) ([A-Z][a-z]+)", "sherlock"]); + + let expected = "\ +For the Watsons, Doctor of this world, as opposed to the Sherlock +be, to a very large extent, the result of luck. Holmes, Sherlock +but Watson, Doctor has to have it taken out for him and dusted, +"; + eqnice!(expected, cmd.stdout()); +}); + +rgtest!(replace_named_groups, |dir: Dir, mut cmd: TestCommand| { + dir.create("sherlock", SHERLOCK); + cmd.args(&[ + "-r", + "$last, $first", + "(?P[A-Z][a-z]+) (?P[A-Z][a-z]+)", + "sherlock", + ]); + + let expected = "\ +For the Watsons, Doctor of this world, as opposed to the Sherlock +be, to a very large extent, the result of luck. Holmes, Sherlock +but Watson, Doctor has to have it taken out for him and dusted, +"; + eqnice!(expected, cmd.stdout()); +}); + +rgtest!(replace_with_only_matching, |dir: Dir, mut cmd: TestCommand| { + dir.create("sherlock", SHERLOCK); + cmd.arg("-o").arg("-r").arg("$1").arg(r"of (\w+)").arg("sherlock"); + + let expected = "\ +this +detective +luck +straw +cigar +"; + eqnice!(expected, cmd.stdout()); +}); + +rgtest!(file_types, |dir: Dir, mut cmd: TestCommand| { + dir.create("sherlock", SHERLOCK); + dir.create("file.py", "Sherlock"); + dir.create("file.rs", "Sherlock"); + cmd.arg("-t").arg("rust").arg("Sherlock"); + + eqnice!("file.rs:Sherlock\n", cmd.stdout()); +}); + +rgtest!(file_types_all, |dir: Dir, mut cmd: TestCommand| { + dir.create("sherlock", SHERLOCK); + dir.create("file.py", "Sherlock"); + cmd.arg("-t").arg("all").arg("Sherlock"); + + eqnice!("file.py:Sherlock\n", cmd.stdout()); +}); + +rgtest!(file_types_negate, |dir: Dir, mut cmd: TestCommand| { + dir.create("sherlock", SHERLOCK); + dir.remove("sherlock"); + dir.create("file.py", "Sherlock"); + dir.create("file.rs", "Sherlock"); + cmd.arg("-T").arg("rust").arg("Sherlock"); + + eqnice!("file.py:Sherlock\n", cmd.stdout()); +}); + +rgtest!(file_types_negate_all, |dir: Dir, mut cmd: TestCommand| { + dir.create("sherlock", SHERLOCK); + dir.create("file.py", "Sherlock"); + cmd.arg("-T").arg("all").arg("Sherlock"); + + let expected = "\ +sherlock:For the Doctor Watsons of this world, as opposed to the Sherlock +sherlock:be, to a very large extent, the result of luck. Sherlock Holmes +"; + eqnice!(expected, cmd.stdout()); +}); + +rgtest!(file_type_clear, |dir: Dir, mut cmd: TestCommand| { + dir.create("sherlock", SHERLOCK); + dir.create("file.py", "Sherlock"); + dir.create("file.rs", "Sherlock"); + cmd.arg("--type-clear").arg("rust").arg("-t").arg("rust").arg("Sherlock"); + + cmd.assert_non_empty_stderr(); +}); + +rgtest!(file_type_add, |dir: Dir, mut cmd: TestCommand| { + dir.create("sherlock", SHERLOCK); + dir.create("file.py", "Sherlock"); + dir.create("file.rs", "Sherlock"); + dir.create("file.wat", "Sherlock"); + cmd.args(&["--type-add", "wat:*.wat", "-t", "wat", "Sherlock"]); + + eqnice!("file.wat:Sherlock\n", cmd.stdout()); +}); + +rgtest!(file_type_add_compose, |dir: Dir, mut cmd: TestCommand| { + dir.create("sherlock", SHERLOCK); + dir.create("file.py", "Sherlock"); + dir.create("file.rs", "Sherlock"); + dir.create("file.wat", "Sherlock"); + cmd.args(&[ + "--type-add", + "wat:*.wat", + "--type-add", + "combo:include:wat,py", + "-t", + "combo", + "Sherlock", + ]); + + let expected = "\ +file.py:Sherlock +file.wat:Sherlock +"; + eqnice!(expected, sort_lines(&cmd.stdout())); +}); + +rgtest!(glob, |dir: Dir, mut cmd: TestCommand| { + dir.create("sherlock", SHERLOCK); + dir.create("file.py", "Sherlock"); + dir.create("file.rs", "Sherlock"); + cmd.arg("-g").arg("*.rs").arg("Sherlock"); + + eqnice!("file.rs:Sherlock\n", cmd.stdout()); +}); + +rgtest!(glob_negate, |dir: Dir, mut cmd: TestCommand| { + dir.create("sherlock", SHERLOCK); + dir.remove("sherlock"); + dir.create("file.py", "Sherlock"); + dir.create("file.rs", "Sherlock"); + cmd.arg("-g").arg("!*.rs").arg("Sherlock"); + + eqnice!("file.py:Sherlock\n", cmd.stdout()); +}); + +rgtest!(glob_case_insensitive, |dir: Dir, mut cmd: TestCommand| { + dir.create("sherlock", SHERLOCK); + dir.create("file.HTML", "Sherlock"); + cmd.arg("--iglob").arg("*.html").arg("Sherlock"); + + eqnice!("file.HTML:Sherlock\n", cmd.stdout()); +}); + +rgtest!(glob_case_sensitive, |dir: Dir, mut cmd: TestCommand| { + dir.create("sherlock", SHERLOCK); + dir.create("file1.HTML", "Sherlock"); + dir.create("file2.html", "Sherlock"); + cmd.arg("--glob").arg("*.html").arg("Sherlock"); + + eqnice!("file2.html:Sherlock\n", cmd.stdout()); +}); + +rgtest!(glob_always_case_insensitive, |dir: Dir, mut cmd: TestCommand| { + dir.create("sherlock", SHERLOCK); + dir.create("file.HTML", "Sherlock"); + cmd.args(&["--glob-case-insensitive", "--glob", "*.html", "Sherlock"]); + + eqnice!("file.HTML:Sherlock\n", cmd.stdout()); +}); + +rgtest!(byte_offset_only_matching, |dir: Dir, mut cmd: TestCommand| { + dir.create("sherlock", SHERLOCK); + cmd.arg("-b").arg("-o").arg("Sherlock"); + + let expected = "\ +sherlock:56:Sherlock +sherlock:177:Sherlock +"; + eqnice!(expected, cmd.stdout()); +}); + +rgtest!(count, |dir: Dir, mut cmd: TestCommand| { + dir.create("sherlock", SHERLOCK); + cmd.arg("--count").arg("Sherlock"); + + let expected = "sherlock:2\n"; + eqnice!(expected, cmd.stdout()); +}); + +rgtest!(count_matches, |dir: Dir, mut cmd: TestCommand| { + dir.create("sherlock", SHERLOCK); + cmd.arg("--count-matches").arg("the"); + + let expected = "sherlock:4\n"; + eqnice!(expected, cmd.stdout()); +}); + +rgtest!(count_matches_inverted, |dir: Dir, mut cmd: TestCommand| { + dir.create("sherlock", SHERLOCK); + cmd.arg("--count-matches").arg("--invert-match").arg("Sherlock"); + + let expected = "sherlock:4\n"; + eqnice!(expected, cmd.stdout()); +}); + +rgtest!(count_matches_via_only, |dir: Dir, mut cmd: TestCommand| { + dir.create("sherlock", SHERLOCK); + cmd.arg("--count").arg("--only-matching").arg("the"); + + let expected = "sherlock:4\n"; + eqnice!(expected, cmd.stdout()); +}); + +rgtest!(include_zero, |dir: Dir, mut cmd: TestCommand| { + dir.create("sherlock", SHERLOCK); + cmd.args(&["--count", "--include-zero", "nada"]); + cmd.assert_err(); + + let output = cmd.raw_output(); + let stdout = String::from_utf8_lossy(&output.stdout); + let expected = "sherlock:0\n"; + + eqnice!(expected, stdout); +}); + +rgtest!(include_zero_override, |dir: Dir, mut cmd: TestCommand| { + dir.create("sherlock", SHERLOCK); + cmd.args(&["--count", "--include-zero", "--no-include-zero", "nada"]); + cmd.assert_err(); + + let output = cmd.raw_output(); + let stdout = String::from_utf8_lossy(&output.stdout); + assert!(stdout.is_empty()); +}); + +rgtest!(files_with_matches, |dir: Dir, mut cmd: TestCommand| { + dir.create("sherlock", SHERLOCK); + cmd.arg("--files-with-matches").arg("Sherlock"); + + let expected = "sherlock\n"; + eqnice!(expected, cmd.stdout()); +}); + +rgtest!(files_without_match, |dir: Dir, mut cmd: TestCommand| { + dir.create("sherlock", SHERLOCK); + dir.create("file.py", "foo"); + cmd.arg("--files-without-match").arg("Sherlock"); + + let expected = "file.py\n"; + eqnice!(expected, cmd.stdout()); +}); + +rgtest!(after_context, |dir: Dir, mut cmd: TestCommand| { + dir.create("sherlock", SHERLOCK); + cmd.arg("-A").arg("1").arg("Sherlock").arg("sherlock"); + + let expected = "\ +For the Doctor Watsons of this world, as opposed to the Sherlock +Holmeses, success in the province of detective work must always +be, to a very large extent, the result of luck. Sherlock Holmes +can extract a clew from a wisp of straw or a flake of cigar ash; +"; + eqnice!(expected, cmd.stdout()); +}); + +rgtest!(after_context_line_numbers, |dir: Dir, mut cmd: TestCommand| { + dir.create("sherlock", SHERLOCK); + cmd.arg("-A").arg("1").arg("-n").arg("Sherlock").arg("sherlock"); + + let expected = "\ +1:For the Doctor Watsons of this world, as opposed to the Sherlock +2-Holmeses, success in the province of detective work must always +3:be, to a very large extent, the result of luck. Sherlock Holmes +4-can extract a clew from a wisp of straw or a flake of cigar ash; +"; + eqnice!(expected, cmd.stdout()); +}); + +rgtest!(before_context, |dir: Dir, mut cmd: TestCommand| { + dir.create("sherlock", SHERLOCK); + cmd.arg("-B").arg("1").arg("Sherlock").arg("sherlock"); + + let expected = "\ +For the Doctor Watsons of this world, as opposed to the Sherlock +Holmeses, success in the province of detective work must always +be, to a very large extent, the result of luck. Sherlock Holmes +"; + eqnice!(expected, cmd.stdout()); +}); + +rgtest!(before_context_line_numbers, |dir: Dir, mut cmd: TestCommand| { + dir.create("sherlock", SHERLOCK); + cmd.arg("-B").arg("1").arg("-n").arg("Sherlock").arg("sherlock"); + + let expected = "\ +1:For the Doctor Watsons of this world, as opposed to the Sherlock +2-Holmeses, success in the province of detective work must always +3:be, to a very large extent, the result of luck. Sherlock Holmes +"; + eqnice!(expected, cmd.stdout()); +}); + +rgtest!(context, |dir: Dir, mut cmd: TestCommand| { + dir.create("sherlock", SHERLOCK); + cmd.arg("-C").arg("1").arg("world|attached").arg("sherlock"); + + let expected = "\ +For the Doctor Watsons of this world, as opposed to the Sherlock +Holmeses, success in the province of detective work must always +-- +but Doctor Watson has to have it taken out for him and dusted, +and exhibited clearly, with a label attached. +"; + eqnice!(expected, cmd.stdout()); +}); + +rgtest!(context_line_numbers, |dir: Dir, mut cmd: TestCommand| { + dir.create("sherlock", SHERLOCK); + cmd.arg("-C").arg("1").arg("-n").arg("world|attached").arg("sherlock"); + + let expected = "\ +1:For the Doctor Watsons of this world, as opposed to the Sherlock +2-Holmeses, success in the province of detective work must always +-- +5-but Doctor Watson has to have it taken out for him and dusted, +6:and exhibited clearly, with a label attached. +"; + eqnice!(expected, cmd.stdout()); +}); + +rgtest!(max_filesize_parse_error_length, |_: Dir, mut cmd: TestCommand| { + cmd.arg("--max-filesize").arg("44444444444444444444"); + cmd.assert_non_empty_stderr(); +}); + +rgtest!(max_filesize_parse_error_suffix, |_: Dir, mut cmd: TestCommand| { + cmd.arg("--max-filesize").arg("45k"); + cmd.assert_non_empty_stderr(); +}); + +rgtest!(max_filesize_parse_no_suffix, |dir: Dir, mut cmd: TestCommand| { + dir.create_size("foo", 40); + dir.create_size("bar", 60); + cmd.arg("--max-filesize").arg("50").arg("--files"); + + eqnice!("foo\n", cmd.stdout()); +}); + +rgtest!(max_filesize_parse_k_suffix, |dir: Dir, mut cmd: TestCommand| { + dir.create_size("foo", 3048); + dir.create_size("bar", 4100); + cmd.arg("--max-filesize").arg("4K").arg("--files"); + + eqnice!("foo\n", cmd.stdout()); +}); + +rgtest!(max_filesize_parse_m_suffix, |dir: Dir, mut cmd: TestCommand| { + dir.create_size("foo", 1000000); + dir.create_size("bar", 1400000); + cmd.arg("--max-filesize").arg("1M").arg("--files"); + + eqnice!("foo\n", cmd.stdout()); +}); + +rgtest!(max_filesize_suffix_overflow, |dir: Dir, mut cmd: TestCommand| { + dir.create_size("foo", 1000000); + + // 2^35 * 2^30 would otherwise overflow + cmd.arg("--max-filesize").arg("34359738368G").arg("--files"); + cmd.assert_non_empty_stderr(); +}); + +rgtest!(ignore_hidden, |dir: Dir, mut cmd: TestCommand| { + dir.create(".sherlock", SHERLOCK); + cmd.arg("Sherlock").assert_err(); +}); + +rgtest!(no_ignore_hidden, |dir: Dir, mut cmd: TestCommand| { + dir.create(".sherlock", SHERLOCK); + cmd.arg("--hidden").arg("Sherlock"); + + let expected = "\ +.sherlock:For the Doctor Watsons of this world, as opposed to the Sherlock +.sherlock:be, to a very large extent, the result of luck. Sherlock Holmes +"; + eqnice!(expected, cmd.stdout()); +}); + +rgtest!(ignore_git, |dir: Dir, mut cmd: TestCommand| { + dir.create("sherlock", SHERLOCK); + dir.create_dir(".git"); + dir.create(".gitignore", "sherlock\n"); + cmd.arg("Sherlock"); + + cmd.assert_err(); +}); + +rgtest!(ignore_generic, |dir: Dir, mut cmd: TestCommand| { + dir.create("sherlock", SHERLOCK); + dir.create(".ignore", "sherlock\n"); + cmd.arg("Sherlock"); + + cmd.assert_err(); +}); + +rgtest!(ignore_ripgrep, |dir: Dir, mut cmd: TestCommand| { + dir.create("sherlock", SHERLOCK); + dir.create(".rgignore", "sherlock\n"); + cmd.arg("Sherlock"); + + cmd.assert_err(); +}); + +rgtest!(no_ignore, |dir: Dir, mut cmd: TestCommand| { + dir.create("sherlock", SHERLOCK); + dir.create(".gitignore", "sherlock\n"); + cmd.arg("--no-ignore").arg("Sherlock"); + + let expected = "\ +sherlock:For the Doctor Watsons of this world, as opposed to the Sherlock +sherlock:be, to a very large extent, the result of luck. Sherlock Holmes +"; + eqnice!(expected, cmd.stdout()); +}); + +rgtest!(ignore_git_parent, |dir: Dir, mut cmd: TestCommand| { + dir.create_dir(".git"); + dir.create(".gitignore", "sherlock\n"); + dir.create_dir("foo"); + dir.create("foo/sherlock", SHERLOCK); + cmd.arg("Sherlock"); + + // Even though we search in foo/, which has no .gitignore, ripgrep will + // traverse parent directories and respect the gitignore files found. + cmd.current_dir("foo"); + cmd.assert_err(); +}); + +rgtest!(ignore_git_parent_stop, |dir: Dir, mut cmd: TestCommand| { + // This tests that searching parent directories for .gitignore files stops + // after it sees a .git directory. To test this, we create this directory + // hierarchy: + // + // .gitignore (contains `sherlock`) + // foo/ + // .git/ + // bar/ + // sherlock + // + // And we perform the search inside `foo/bar/`. ripgrep will stop looking + // for .gitignore files after it sees `foo/.git/`, and therefore not + // respect the top-level `.gitignore` containing `sherlock`. + dir.create(".gitignore", "sherlock\n"); + dir.create_dir("foo"); + dir.create_dir("foo/.git"); + dir.create_dir("foo/bar"); + dir.create("foo/bar/sherlock", SHERLOCK); + cmd.arg("Sherlock"); + cmd.current_dir("foo/bar"); + + let expected = "\ +sherlock:For the Doctor Watsons of this world, as opposed to the Sherlock +sherlock:be, to a very large extent, the result of luck. Sherlock Holmes +"; + eqnice!(expected, cmd.stdout()); +}); + +// Like ignore_git_parent_stop, but with a .git file instead of a .git +// directory. +rgtest!(ignore_git_parent_stop_file, |dir: Dir, mut cmd: TestCommand| { + // This tests that searching parent directories for .gitignore files stops + // after it sees a .git *file*. A .git file is used for submodules. To test + // this, we create this directory hierarchy: + // + // .gitignore (contains `sherlock`) + // foo/ + // .git + // bar/ + // sherlock + // + // And we perform the search inside `foo/bar/`. ripgrep will stop looking + // for .gitignore files after it sees `foo/.git`, and therefore not + // respect the top-level `.gitignore` containing `sherlock`. + dir.create(".gitignore", "sherlock\n"); + dir.create_dir("foo"); + dir.create("foo/.git", ""); + dir.create_dir("foo/bar"); + dir.create("foo/bar/sherlock", SHERLOCK); + cmd.arg("Sherlock"); + cmd.current_dir("foo/bar"); + + let expected = "\ +sherlock:For the Doctor Watsons of this world, as opposed to the Sherlock +sherlock:be, to a very large extent, the result of luck. Sherlock Holmes +"; + eqnice!(expected, cmd.stdout()); +}); + +rgtest!(ignore_ripgrep_parent_no_stop, |dir: Dir, mut cmd: TestCommand| { + // This is like the `ignore_git_parent_stop` test, except it checks that + // ripgrep *doesn't* stop checking for .rgignore files. + dir.create(".rgignore", "sherlock\n"); + dir.create_dir("foo"); + dir.create_dir("foo/.git"); + dir.create_dir("foo/bar"); + dir.create("foo/bar/sherlock", SHERLOCK); + cmd.arg("Sherlock"); + cmd.current_dir("foo/bar"); + + // The top-level .rgignore applies. + cmd.assert_err(); +}); + +rgtest!(no_parent_ignore_git, |dir: Dir, mut cmd: TestCommand| { + // Set up a directory hierarchy like this: + // + // .git/ + // .gitignore + // foo/ + // .gitignore + // sherlock + // watson + // + // Where `.gitignore` contains `sherlock` and `foo/.gitignore` contains + // `watson`. + // + // Now *do the search* from the foo directory. By default, ripgrep will + // search parent directories for .gitignore files. The --no-ignore-parent + // flag should prevent that. At the same time, the `foo/.gitignore` file + // will still be respected (since the search is happening in `foo/`). + // + // In other words, we should only see results from `sherlock`, not from + // `watson`. + dir.create_dir(".git"); + dir.create(".gitignore", "sherlock\n"); + dir.create_dir("foo"); + dir.create("foo/.gitignore", "watson\n"); + dir.create("foo/sherlock", SHERLOCK); + dir.create("foo/watson", SHERLOCK); + cmd.arg("--no-ignore-parent").arg("Sherlock"); + cmd.current_dir("foo"); + + let expected = "\ +sherlock:For the Doctor Watsons of this world, as opposed to the Sherlock +sherlock:be, to a very large extent, the result of luck. Sherlock Holmes +"; + eqnice!(expected, cmd.stdout()); +}); + +// Multiple explicit search roots must apply parent .gitignore rules using each +// root's own path base. Searching `src` before `tests` used to leak the wrong +// absolute base into cached parent matchers so `src/invalid` was not ignored. +// See: https://github.com/BurntSushi/ripgrep/issues/3376 +rgtest!(ignore_git_multi_root_order, |dir: Dir, mut cmd: TestCommand| { + dir.create_dir(".git"); + dir.create(".gitignore", "src/invalid\n"); + dir.create_dir("src"); + dir.create_dir("tests"); + dir.create("src/invalid", "this\n"); + dir.create("src/valid", "this\n"); + dir.create("tests/valid", "this\n"); + + cmd.args(&["--files-with-matches", "this", "src", "tests"]); + let got = cmd.stdout(); + assert!( + !got.contains("invalid"), + "src/invalid must stay ignored with multiple roots, got:\n{}", + got + ); + assert!(got.contains("src/valid"), "missing src/valid in:\n{}", got); + assert!(got.contains("tests/valid"), "missing tests/valid in:\n{}", got); + + // Reverse CLI order should behave the same. + let mut cmd = dir.command(); + cmd.args(&["--files-with-matches", "this", "tests", "src"]); + let got = cmd.stdout(); + assert!( + !got.contains("invalid"), + "src/invalid must stay ignored with roots reversed, got:\n{}", + got + ); +}); + +// Same multi-root path-base issue for `.rgignore`. +// See: https://github.com/BurntSushi/ripgrep/issues/3320 +rgtest!(ignore_rgignore_multi_root_order, |dir: Dir, mut cmd: TestCommand| { + dir.create(".rgignore", "beta/**/*.svg\n"); + dir.create_dir("alpha"); + dir.create_dir("beta"); + dir.create("alpha/a.txt", "AWS\n"); + dir.create("beta/x.svg", "AWS\n"); + + cmd.args(&["--files-with-matches", "AWS", "alpha", "beta"]); + let got = cmd.stdout(); + assert!( + !got.contains("x.svg"), + "beta/x.svg must be ignored with multiple roots, got:\n{}", + got + ); + assert!(got.contains("alpha/a.txt"), "missing alpha/a.txt in:\n{}", got); + + let mut cmd = dir.command(); + cmd.args(&["--files-with-matches", "AWS", "beta", "alpha"]); + let got = cmd.stdout(); + assert!( + !got.contains("x.svg"), + "beta/x.svg must be ignored with roots reversed, got:\n{}", + got + ); +}); + +rgtest!(symlink_nofollow, |dir: Dir, mut cmd: TestCommand| { + dir.create_dir("foo"); + dir.create_dir("foo/bar"); + dir.link_dir("foo/baz", "foo/bar/baz"); + dir.create_dir("foo/baz"); + dir.create("foo/baz/sherlock", SHERLOCK); + cmd.arg("Sherlock"); + cmd.current_dir("foo/bar"); + + cmd.assert_err(); +}); + +#[cfg(not(windows))] +rgtest!(symlink_follow, |dir: Dir, mut cmd: TestCommand| { + dir.create_dir("foo"); + dir.create_dir("foo/bar"); + dir.create_dir("foo/baz"); + dir.create("foo/baz/sherlock", SHERLOCK); + dir.link_dir("foo/baz", "foo/bar/baz"); + cmd.arg("-L").arg("Sherlock"); + cmd.current_dir("foo/bar"); + + let expected = "\ +baz/sherlock:For the Doctor Watsons of this world, as opposed to the Sherlock +baz/sherlock:be, to a very large extent, the result of luck. Sherlock Holmes +"; + eqnice!(expected, cmd.stdout()); +}); + +rgtest!(unrestricted1, |dir: Dir, mut cmd: TestCommand| { + dir.create("sherlock", SHERLOCK); + dir.create(".gitignore", "sherlock\n"); + cmd.arg("-u").arg("Sherlock"); + + let expected = "\ +sherlock:For the Doctor Watsons of this world, as opposed to the Sherlock +sherlock:be, to a very large extent, the result of luck. Sherlock Holmes +"; + eqnice!(expected, cmd.stdout()); +}); + +rgtest!(unrestricted2, |dir: Dir, mut cmd: TestCommand| { + dir.create(".sherlock", SHERLOCK); + cmd.arg("-uu").arg("Sherlock"); + + let expected = "\ +.sherlock:For the Doctor Watsons of this world, as opposed to the Sherlock +.sherlock:be, to a very large extent, the result of luck. Sherlock Holmes +"; + eqnice!(expected, cmd.stdout()); +}); + +rgtest!(unrestricted3, |dir: Dir, mut cmd: TestCommand| { + dir.create("sherlock", SHERLOCK); + dir.create("hay", "foo\x00bar\nfoo\x00baz\n"); + cmd.arg("-uuu").arg("foo"); + + let expected = "\ +hay: binary file matches (found \"\\0\" byte around offset 3) +"; + eqnice!(expected, cmd.stdout()); +}); + +rgtest!(vimgrep, |dir: Dir, mut cmd: TestCommand| { + dir.create("sherlock", SHERLOCK); + cmd.arg("--vimgrep").arg("Sherlock|Watson"); + + let expected = "\ +sherlock:1:16:For the Doctor Watsons of this world, as opposed to the Sherlock +sherlock:1:57:For the Doctor Watsons of this world, as opposed to the Sherlock +sherlock:3:49:be, to a very large extent, the result of luck. Sherlock Holmes +sherlock:5:12:but Doctor Watson has to have it taken out for him and dusted, +"; + eqnice!(expected, cmd.stdout()); +}); + +rgtest!(vimgrep_no_line, |dir: Dir, mut cmd: TestCommand| { + dir.create("sherlock", SHERLOCK); + cmd.arg("--vimgrep").arg("-N").arg("Sherlock|Watson"); + + let expected = "\ +sherlock:16:For the Doctor Watsons of this world, as opposed to the Sherlock +sherlock:57:For the Doctor Watsons of this world, as opposed to the Sherlock +sherlock:49:be, to a very large extent, the result of luck. Sherlock Holmes +sherlock:12:but Doctor Watson has to have it taken out for him and dusted, +"; + eqnice!(expected, cmd.stdout()); +}); + +rgtest!(vimgrep_no_line_no_column, |dir: Dir, mut cmd: TestCommand| { + dir.create("sherlock", SHERLOCK); + cmd.arg("--vimgrep").arg("-N").arg("--no-column").arg("Sherlock|Watson"); + + let expected = "\ +sherlock:For the Doctor Watsons of this world, as opposed to the Sherlock +sherlock:For the Doctor Watsons of this world, as opposed to the Sherlock +sherlock:be, to a very large extent, the result of luck. Sherlock Holmes +sherlock:but Doctor Watson has to have it taken out for him and dusted, +"; + eqnice!(expected, cmd.stdout()); +}); + +rgtest!(preprocessing, |dir: Dir, mut cmd: TestCommand| { + if !cmd_exists("xzcat") { + return; + } + + dir.create_bytes("sherlock.xz", include_bytes!("./data/sherlock.xz")); + cmd.arg("--pre").arg("xzcat").arg("Sherlock").arg("sherlock.xz"); + + let expected = "\ +For the Doctor Watsons of this world, as opposed to the Sherlock +be, to a very large extent, the result of luck. Sherlock Holmes +"; + eqnice!(expected, cmd.stdout()); +}); + +rgtest!(preprocessing_glob, |dir: Dir, mut cmd: TestCommand| { + if !cmd_exists("xzcat") { + return; + } + + dir.create("sherlock", SHERLOCK); + dir.create_bytes("sherlock.xz", include_bytes!("./data/sherlock.xz")); + cmd.args(&["--pre", "xzcat", "--pre-glob", "*.xz", "Sherlock"]); + + let expected = "\ +sherlock.xz:For the Doctor Watsons of this world, as opposed to the Sherlock +sherlock.xz:be, to a very large extent, the result of luck. Sherlock Holmes +sherlock:For the Doctor Watsons of this world, as opposed to the Sherlock +sherlock:be, to a very large extent, the result of luck. Sherlock Holmes +"; + eqnice!(sort_lines(expected), sort_lines(&cmd.stdout())); +}); + +rgtest!(compressed_gzip, |dir: Dir, mut cmd: TestCommand| { + if !cmd_exists("gzip") { + return; + } + + dir.create_bytes("sherlock.gz", include_bytes!("./data/sherlock.gz")); + cmd.arg("-z").arg("Sherlock").arg("sherlock.gz"); + + let expected = "\ +For the Doctor Watsons of this world, as opposed to the Sherlock +be, to a very large extent, the result of luck. Sherlock Holmes +"; + eqnice!(expected, cmd.stdout()); +}); + +rgtest!(compressed_bzip2, |dir: Dir, mut cmd: TestCommand| { + if !cmd_exists("bzip2") { + return; + } + + dir.create_bytes("sherlock.bz2", include_bytes!("./data/sherlock.bz2")); + cmd.arg("-z").arg("Sherlock").arg("sherlock.bz2"); + + let expected = "\ +For the Doctor Watsons of this world, as opposed to the Sherlock +be, to a very large extent, the result of luck. Sherlock Holmes +"; + eqnice!(expected, cmd.stdout()); +}); + +rgtest!(compressed_xz, |dir: Dir, mut cmd: TestCommand| { + if !cmd_exists("xz") { + return; + } + + dir.create_bytes("sherlock.xz", include_bytes!("./data/sherlock.xz")); + cmd.arg("-z").arg("Sherlock").arg("sherlock.xz"); + + let expected = "\ +For the Doctor Watsons of this world, as opposed to the Sherlock +be, to a very large extent, the result of luck. Sherlock Holmes +"; + eqnice!(expected, cmd.stdout()); +}); + +rgtest!(compressed_lz4, |dir: Dir, mut cmd: TestCommand| { + if !cmd_exists("lz4") { + return; + } + + dir.create_bytes("sherlock.lz4", include_bytes!("./data/sherlock.lz4")); + cmd.arg("-z").arg("Sherlock").arg("sherlock.lz4"); + + let expected = "\ +For the Doctor Watsons of this world, as opposed to the Sherlock +be, to a very large extent, the result of luck. Sherlock Holmes +"; + eqnice!(expected, cmd.stdout()); +}); + +rgtest!(compressed_lzma, |dir: Dir, mut cmd: TestCommand| { + if !cmd_exists("xz") { + return; + } + + dir.create_bytes("sherlock.lzma", include_bytes!("./data/sherlock.lzma")); + cmd.arg("-z").arg("Sherlock").arg("sherlock.lzma"); + + let expected = "\ +For the Doctor Watsons of this world, as opposed to the Sherlock +be, to a very large extent, the result of luck. Sherlock Holmes +"; + eqnice!(expected, cmd.stdout()); +}); + +rgtest!(compressed_brotli, |dir: Dir, mut cmd: TestCommand| { + if !cmd_exists("brotli") { + return; + } + + dir.create_bytes("sherlock.br", include_bytes!("./data/sherlock.br")); + cmd.arg("-z").arg("Sherlock").arg("sherlock.br"); + + let expected = "\ +For the Doctor Watsons of this world, as opposed to the Sherlock +be, to a very large extent, the result of luck. Sherlock Holmes +"; + eqnice!(expected, cmd.stdout()); +}); + +rgtest!(compressed_zstd, |dir: Dir, mut cmd: TestCommand| { + if !cmd_exists("zstd") { + return; + } + + dir.create_bytes("sherlock.zst", include_bytes!("./data/sherlock.zst")); + cmd.arg("-z").arg("Sherlock").arg("sherlock.zst"); + + let expected = "\ +For the Doctor Watsons of this world, as opposed to the Sherlock +be, to a very large extent, the result of luck. Sherlock Holmes +"; + eqnice!(expected, cmd.stdout()); +}); + +rgtest!(compressed_uncompress, |dir: Dir, mut cmd: TestCommand| { + if !cmd_exists("uncompress") { + return; + } + + dir.create_bytes("sherlock.Z", include_bytes!("./data/sherlock.Z")); + cmd.arg("-z").arg("Sherlock").arg("sherlock.Z"); + + let expected = "\ + For the Doctor Watsons of this world, as opposed to the Sherlock +be, to a very large extent, the result of luck. Sherlock Holmes +"; + eqnice!(expected, cmd.stdout()); +}); + +rgtest!(compressed_failing_gzip, |dir: Dir, mut cmd: TestCommand| { + if !cmd_exists("gzip") { + return; + } + + dir.create("sherlock.gz", SHERLOCK); + cmd.arg("-z").arg("Sherlock").arg("sherlock.gz"); + + cmd.assert_non_empty_stderr(); +}); + +rgtest!(binary_convert, |dir: Dir, mut cmd: TestCommand| { + dir.create("file", "foo\x00bar\nfoo\x00baz\n"); + cmd.arg("--no-mmap").arg("foo").arg("file"); + + let expected = "\ +binary file matches (found \"\\0\" byte around offset 3) +"; + eqnice!(expected, cmd.stdout()); +}); + +rgtest!(binary_convert_mmap, |dir: Dir, mut cmd: TestCommand| { + dir.create("file", "foo\x00bar\nfoo\x00baz\n"); + cmd.arg("--mmap").arg("foo").arg("file"); + + let expected = "\ +binary file matches (found \"\\0\" byte around offset 3) +"; + eqnice!(expected, cmd.stdout()); +}); + +rgtest!(binary_quit, |dir: Dir, mut cmd: TestCommand| { + dir.create("file", "foo\x00bar\nfoo\x00baz\n"); + cmd.arg("--no-mmap").arg("foo").arg("-gfile"); + cmd.assert_err(); +}); + +rgtest!(binary_quit_mmap, |dir: Dir, mut cmd: TestCommand| { + dir.create("file", "foo\x00bar\nfoo\x00baz\n"); + cmd.arg("--mmap").arg("foo").arg("-gfile"); + cmd.assert_err(); +}); + +// The following two tests show a discrepancy in search results between +// searching with memory mapped files and stream searching. Stream searching +// uses a heuristic (that GNU grep also uses) where NUL bytes are replaced with +// the EOL terminator, which tends to avoid allocating large amounts of memory +// for really long "lines." The memory map searcher has no need to worry about +// such things, and more than that, it would be pretty hard for it to match the +// semantics of streaming search in this case. +// +// Binary files with lots of NULs aren't really part of the use case of ripgrep +// (or any other grep-like tool for that matter), so we shouldn't feel too bad +// about it. +rgtest!(binary_search_mmap, |dir: Dir, mut cmd: TestCommand| { + dir.create("file", "foo\x00bar\nfoo\x00baz\n"); + cmd.arg("-a").arg("--mmap").arg("foo").arg("file"); + eqnice!("foo\x00bar\nfoo\x00baz\n", cmd.stdout()); +}); + +rgtest!(binary_search_no_mmap, |dir: Dir, mut cmd: TestCommand| { + dir.create("file", "foo\x00bar\nfoo\x00baz\n"); + cmd.arg("-a").arg("--no-mmap").arg("foo").arg("file"); + eqnice!("foo\x00bar\nfoo\x00baz\n", cmd.stdout()); +}); + +rgtest!(files, |dir: Dir, mut cmd: TestCommand| { + dir.create("file", ""); + dir.create_dir("dir"); + dir.create("dir/file", ""); + cmd.arg("--files"); + + eqnice!(sort_lines("file\ndir/file\n"), sort_lines(&cmd.stdout())); +}); + +rgtest!(type_list, |_: Dir, mut cmd: TestCommand| { + cmd.arg("--type-list"); + // This can change over time, so just make sure we print something. + assert!(!cmd.stdout().is_empty()); +}); + +// The following series of tests seeks to test all permutations of ripgrep's +// sorted queries. +// +// They all rely on this setup function, which sets up this particular file +// structure with a particular creation order: +// ├── a # 1 +// ├── b # 4 +// └── dir # 2 +// ├── c # 3 +// └── d # 5 +// +// This order is important when sorting them by system time-stamps. +fn sort_setup(dir: Dir) { + use std::{thread::sleep, time::Duration}; + // As reported in https://github.com/BurntSushi/ripgrep/issues/3071 + // this test fails if sufficient delay is not given on Windows/Aarch64. + let delay = if cfg!(all(windows, target_arch = "aarch64")) { + Duration::from_millis(1000) + } else { + Duration::from_millis(100) + }; + let sub_dir = dir.path().join("dir"); + dir.create("a", "test"); + sleep(delay); + dir.create_dir(&sub_dir); + sleep(delay); + dir.create(sub_dir.join("c"), "test"); + sleep(delay); + dir.create("b", "test"); + sleep(delay); + dir.create(sub_dir.join("d"), "test"); +} + +rgtest!(sort_files, |dir: Dir, mut cmd: TestCommand| { + sort_setup(dir); + let expected = "a:test\nb:test\ndir/c:test\ndir/d:test\n"; + eqnice!(expected, cmd.args(["--sort", "path", "test"]).stdout()); +}); + +rgtest!(sort_accessed, |dir: Dir, mut cmd: TestCommand| { + if crate::util::is_cross() { + return; + } + sort_setup(dir); + let expected = "a:test\ndir/c:test\nb:test\ndir/d:test\n"; + eqnice!(expected, cmd.args(["--sort", "accessed", "test"]).stdout()); +}); + +rgtest!(sortr_accessed, |dir: Dir, mut cmd: TestCommand| { + if crate::util::is_cross() { + return; + } + sort_setup(dir); + let expected = "dir/d:test\nb:test\ndir/c:test\na:test\n"; + eqnice!(expected, cmd.args(["--sortr", "accessed", "test"]).stdout()); +}); diff --git a/userland/ripgrep/tests/multiline.rs b/userland/ripgrep/tests/multiline.rs new file mode 100644 index 0000000..d084c96 --- /dev/null +++ b/userland/ripgrep/tests/multiline.rs @@ -0,0 +1,121 @@ +use crate::hay::SHERLOCK; +use crate::util::{Dir, TestCommand}; + +// This tests that multiline matches that span multiple lines, but where +// multiple matches may begin and end on the same line work correctly. +rgtest!(overlap1, |dir: Dir, mut cmd: TestCommand| { + dir.create("test", "xxx\nabc\ndefxxxabc\ndefxxx\nxxx"); + cmd.arg("-n").arg("-U").arg("abc\ndef").arg("test"); + eqnice!("2:abc\n3:defxxxabc\n4:defxxx\n", cmd.stdout()); +}); + +// Like overlap1, but tests the case where one match ends at precisely the same +// location at which the next match begins. +rgtest!(overlap2, |dir: Dir, mut cmd: TestCommand| { + dir.create("test", "xxx\nabc\ndefabc\ndefxxx\nxxx"); + cmd.arg("-n").arg("-U").arg("abc\ndef").arg("test"); + eqnice!("2:abc\n3:defabc\n4:defxxx\n", cmd.stdout()); +}); + +// Tests that even in a multiline search, a '.' does not match a newline. +rgtest!(dot_no_newline, |dir: Dir, mut cmd: TestCommand| { + dir.create("sherlock", SHERLOCK); + cmd.args(&["-n", "-U", "of this world.+detective work", "sherlock"]); + cmd.assert_err(); +}); + +// Tests that the --multiline-dotall flag causes '.' to match a newline. +rgtest!(dot_all, |dir: Dir, mut cmd: TestCommand| { + dir.create("sherlock", SHERLOCK); + cmd.args(&[ + "-n", + "-U", + "--multiline-dotall", + "of this world.+detective work", + "sherlock", + ]); + + let expected = "\ +1:For the Doctor Watsons of this world, as opposed to the Sherlock +2:Holmeses, success in the province of detective work must always +"; + eqnice!(expected, cmd.stdout()); +}); + +// Tests that --only-matching works in multiline mode. +rgtest!(only_matching, |dir: Dir, mut cmd: TestCommand| { + dir.create("sherlock", SHERLOCK); + cmd.args(&[ + "-n", + "-U", + "--only-matching", + r"Watson|Sherlock\p{Any}+?Holmes", + "sherlock", + ]); + + let expected = "\ +1:Watson +1:Sherlock +2:Holmes +3:Sherlock Holmes +5:Watson +"; + eqnice!(expected, cmd.stdout()); +}); + +// Tests that --vimgrep works in multiline mode. +// +// In particular, we test that only the first line of each match is printed, +// even when a match spans multiple lines. +// +// See: https://github.com/BurntSushi/ripgrep/issues/1866 +rgtest!(vimgrep, |dir: Dir, mut cmd: TestCommand| { + dir.create("sherlock", SHERLOCK); + cmd.args(&[ + "-n", + "-U", + "--vimgrep", + r"Watson|Sherlock\p{Any}+?Holmes", + "sherlock", + ]); + + let expected = "\ +sherlock:1:16:For the Doctor Watsons of this world, as opposed to the Sherlock +sherlock:1:57:For the Doctor Watsons of this world, as opposed to the Sherlock +sherlock:3:49:be, to a very large extent, the result of luck. Sherlock Holmes +sherlock:5:12:but Doctor Watson has to have it taken out for him and dusted, +"; + eqnice!(expected, cmd.stdout()); +}); + +// Tests that multiline search works when reading from stdin. This is an +// important test because multiline search must read the entire contents of +// what it is searching into memory before executing the search. +rgtest!(stdin, |_: Dir, mut cmd: TestCommand| { + cmd.args(&["-n", "-U", r"of this world\p{Any}+?detective work"]); + let expected = "\ +1:For the Doctor Watsons of this world, as opposed to the Sherlock +2:Holmeses, success in the province of detective work must always +"; + eqnice!(expected, cmd.pipe(SHERLOCK.as_bytes())); +}); + +// Test that multiline search and contextual matches work. +rgtest!(context, |dir: Dir, mut cmd: TestCommand| { + dir.create("sherlock", SHERLOCK); + cmd.args(&[ + "-n", + "-U", + "-C1", + r"detective work\p{Any}+?result of luck", + "sherlock", + ]); + + let expected = "\ +1-For the Doctor Watsons of this world, as opposed to the Sherlock +2:Holmeses, success in the province of detective work must always +3:be, to a very large extent, the result of luck. Sherlock Holmes +4-can extract a clew from a wisp of straw or a flake of cigar ash; +"; + eqnice!(expected, cmd.stdout()); +}); diff --git a/userland/ripgrep/tests/regression.rs b/userland/ripgrep/tests/regression.rs new file mode 100644 index 0000000..dcca01f --- /dev/null +++ b/userland/ripgrep/tests/regression.rs @@ -0,0 +1,1744 @@ +use crate::hay::SHERLOCK; +use crate::util::{Dir, TestCommand, sort_lines}; + +// See: https://github.com/BurntSushi/ripgrep/issues/16 +rgtest!(r16, |dir: Dir, mut cmd: TestCommand| { + dir.create_dir(".git"); + dir.create(".gitignore", "ghi/"); + dir.create_dir("ghi"); + dir.create_dir("def/ghi"); + dir.create("ghi/toplevel.txt", "xyz"); + dir.create("def/ghi/subdir.txt", "xyz"); + + cmd.arg("xyz").assert_err(); +}); + +// See: https://github.com/BurntSushi/ripgrep/issues/25 +rgtest!(r25, |dir: Dir, mut cmd: TestCommand| { + dir.create_dir(".git"); + dir.create(".gitignore", "/llvm/"); + dir.create_dir("src/llvm"); + dir.create("src/llvm/foo", "test"); + + cmd.arg("test"); + eqnice!("src/llvm/foo:test\n", cmd.stdout()); + + cmd.current_dir("src"); + eqnice!("llvm/foo:test\n", cmd.stdout()); +}); + +// See: https://github.com/BurntSushi/ripgrep/issues/30 +rgtest!(r30, |dir: Dir, mut cmd: TestCommand| { + dir.create(".gitignore", "vendor/**\n!vendor/manifest"); + dir.create_dir("vendor"); + dir.create("vendor/manifest", "test"); + + eqnice!("vendor/manifest:test\n", cmd.arg("test").stdout()); +}); + +// See: https://github.com/BurntSushi/ripgrep/issues/49 +rgtest!(r49, |dir: Dir, mut cmd: TestCommand| { + dir.create(".gitignore", "foo/bar"); + dir.create_dir("test/foo/bar"); + dir.create("test/foo/bar/baz", "test"); + + cmd.arg("xyz").assert_err(); +}); + +// See: https://github.com/BurntSushi/ripgrep/issues/50 +rgtest!(r50, |dir: Dir, mut cmd: TestCommand| { + dir.create(".gitignore", "XXX/YYY/"); + dir.create_dir("abc/def/XXX/YYY"); + dir.create_dir("ghi/XXX/YYY"); + dir.create("abc/def/XXX/YYY/bar", "test"); + dir.create("ghi/XXX/YYY/bar", "test"); + + cmd.arg("xyz").assert_err(); +}); + +// See: https://github.com/BurntSushi/ripgrep/issues/64 +rgtest!(r64, |dir: Dir, mut cmd: TestCommand| { + dir.create_dir("dir"); + dir.create_dir("foo"); + dir.create("dir/abc", ""); + dir.create("foo/abc", ""); + + eqnice!("foo/abc\n", cmd.arg("--files").arg("foo").stdout()); +}); + +// See: https://github.com/BurntSushi/ripgrep/issues/65 +rgtest!(r65, |dir: Dir, mut cmd: TestCommand| { + dir.create_dir(".git"); + dir.create(".gitignore", "a/"); + dir.create_dir("a"); + dir.create("a/foo", "xyz"); + dir.create("a/bar", "xyz"); + + cmd.arg("xyz").assert_err(); +}); + +// See: https://github.com/BurntSushi/ripgrep/issues/67 +rgtest!(r67, |dir: Dir, mut cmd: TestCommand| { + dir.create_dir(".git"); + dir.create(".gitignore", "/*\n!/dir"); + dir.create_dir("dir"); + dir.create_dir("foo"); + dir.create("foo/bar", "test"); + dir.create("dir/bar", "test"); + + eqnice!("dir/bar:test\n", cmd.arg("test").stdout()); +}); + +// See: https://github.com/BurntSushi/ripgrep/issues/87 +rgtest!(r87, |dir: Dir, mut cmd: TestCommand| { + dir.create_dir(".git"); + dir.create(".gitignore", "foo\n**no-vcs**"); + dir.create("foo", "test"); + + cmd.arg("test").assert_err(); +}); + +// See: https://github.com/BurntSushi/ripgrep/issues/90 +rgtest!(r90, |dir: Dir, mut cmd: TestCommand| { + dir.create_dir(".git"); + dir.create(".gitignore", "!.foo"); + dir.create(".foo", "test"); + + eqnice!(".foo:test\n", cmd.arg("test").stdout()); +}); + +// See: https://github.com/BurntSushi/ripgrep/issues/93 +rgtest!(r93, |dir: Dir, mut cmd: TestCommand| { + dir.create("foo", "192.168.1.1"); + + eqnice!("foo:192.168.1.1\n", cmd.arg(r"(\d{1,3}\.){3}\d{1,3}").stdout()); +}); + +// See: https://github.com/BurntSushi/ripgrep/issues/99 +rgtest!(r99, |dir: Dir, mut cmd: TestCommand| { + dir.create("foo1", "test"); + dir.create("foo2", "zzz"); + dir.create("bar", "test"); + + eqnice!( + sort_lines("bar\ntest\n\nfoo1\ntest\n"), + sort_lines(&cmd.arg("-j1").arg("--heading").arg("test").stdout()) + ); +}); + +// See: https://github.com/BurntSushi/ripgrep/issues/105 +rgtest!(r105_part1, |dir: Dir, mut cmd: TestCommand| { + dir.create("foo", "zztest"); + + eqnice!("foo:1:3:zztest\n", cmd.arg("--vimgrep").arg("test").stdout()); +}); + +// See: https://github.com/BurntSushi/ripgrep/issues/105 +rgtest!(r105_part2, |dir: Dir, mut cmd: TestCommand| { + dir.create("foo", "zztest"); + + eqnice!("foo:1:3:zztest\n", cmd.arg("--column").arg("test").stdout()); +}); + +// See: https://github.com/BurntSushi/ripgrep/issues/127 +rgtest!(r127, |dir: Dir, mut cmd: TestCommand| { + // Set up a directory hierarchy like this: + // + // .gitignore + // foo/ + // sherlock + // watson + // + // Where `.gitignore` contains `foo/sherlock`. + // + // ripgrep should ignore 'foo/sherlock' giving us results only from + // 'foo/watson' but on Windows ripgrep will include both 'foo/sherlock' and + // 'foo/watson' in the search results. + dir.create_dir(".git"); + dir.create(".gitignore", "foo/sherlock\n"); + dir.create_dir("foo"); + dir.create("foo/sherlock", SHERLOCK); + dir.create("foo/watson", SHERLOCK); + + let expected = "\ +foo/watson:For the Doctor Watsons of this world, as opposed to the Sherlock +foo/watson:be, to a very large extent, the result of luck. Sherlock Holmes +"; + assert_eq!(expected, cmd.arg("Sherlock").stdout()); +}); + +// See: https://github.com/BurntSushi/ripgrep/issues/128 +rgtest!(r128, |dir: Dir, mut cmd: TestCommand| { + dir.create_bytes("foo", b"01234567\x0b\n\x0b\n\x0b\n\x0b\nx"); + + eqnice!("foo:5:x\n", cmd.arg("-n").arg("x").stdout()); +}); + +// See: https://github.com/BurntSushi/ripgrep/issues/131 +// +// TODO(burntsushi): Darwin doesn't like this test for some reason. Probably +// due to the weird file path. +#[cfg(not(target_os = "macos"))] +rgtest!(r131, |dir: Dir, mut cmd: TestCommand| { + dir.create_dir(".git"); + dir.create(".gitignore", "TopÑapa"); + dir.create("TopÑapa", "test"); + + cmd.arg("test").assert_err(); +}); + +// See: https://github.com/BurntSushi/ripgrep/issues/137 +// +// TODO(burntsushi): Figure out how to make this test work on Windows. Right +// now it gives "access denied" errors when trying to create a file symlink. +// For now, disable test on Windows. +#[cfg(not(windows))] +rgtest!(r137, |dir: Dir, mut cmd: TestCommand| { + dir.create("sherlock", SHERLOCK); + dir.link_file("sherlock", "sym1"); + dir.link_file("sherlock", "sym2"); + + let expected = "\ +./sherlock:For the Doctor Watsons of this world, as opposed to the Sherlock +./sherlock:be, to a very large extent, the result of luck. Sherlock Holmes +sym1:For the Doctor Watsons of this world, as opposed to the Sherlock +sym1:be, to a very large extent, the result of luck. Sherlock Holmes +sym2:For the Doctor Watsons of this world, as opposed to the Sherlock +sym2:be, to a very large extent, the result of luck. Sherlock Holmes +"; + cmd.arg("-j1").arg("Sherlock").arg("./").arg("sym1").arg("sym2"); + eqnice!(expected, cmd.stdout()); +}); + +// See: https://github.com/BurntSushi/ripgrep/issues/156 +rgtest!(r156, |dir: Dir, mut cmd: TestCommand| { + let expected = r#"#parse('widgets/foo_bar_macros.vm') +#parse ( 'widgets/mobile/foo_bar_macros.vm' ) +#parse ("widgets/foobarhiddenformfields.vm") +#parse ( "widgets/foo_bar_legal.vm" ) +#include( 'widgets/foo_bar_tips.vm' ) +#include('widgets/mobile/foo_bar_macros.vm') +#include ("widgets/mobile/foo_bar_resetpw.vm") +#parse('widgets/foo-bar-macros.vm') +#parse ( 'widgets/mobile/foo-bar-macros.vm' ) +#parse ("widgets/foo-bar-hiddenformfields.vm") +#parse ( "widgets/foo-bar-legal.vm" ) +#include( 'widgets/foo-bar-tips.vm' ) +#include('widgets/mobile/foo-bar-macros.vm') +#include ("widgets/mobile/foo-bar-resetpw.vm") +"#; + dir.create("testcase.txt", expected); + + cmd.arg("-N"); + cmd.arg(r#"#(?:parse|include)\s*\(\s*(?:"|')[./A-Za-z_-]+(?:"|')"#); + cmd.arg("testcase.txt"); + eqnice!(expected, cmd.stdout()); +}); + +// See: https://github.com/BurntSushi/ripgrep/issues/184 +rgtest!(r184, |dir: Dir, mut cmd: TestCommand| { + dir.create(".gitignore", ".*"); + dir.create_dir("foo/bar"); + dir.create("foo/bar/baz", "test"); + + cmd.arg("test"); + eqnice!("foo/bar/baz:test\n", cmd.stdout()); + + cmd.current_dir("./foo/bar"); + eqnice!("baz:test\n", cmd.stdout()); +}); + +// See: https://github.com/BurntSushi/ripgrep/issues/199 +rgtest!(r199, |dir: Dir, mut cmd: TestCommand| { + dir.create("foo", "tEsT"); + + eqnice!("foo:tEsT\n", cmd.arg("--smart-case").arg(r"\btest\b").stdout()); +}); + +// See: https://github.com/BurntSushi/ripgrep/issues/206 +rgtest!(r206, |dir: Dir, mut cmd: TestCommand| { + dir.create_dir("foo"); + dir.create("foo/bar.txt", "test"); + + cmd.arg("test").arg("-g").arg("*.txt"); + eqnice!("foo/bar.txt:test\n", cmd.stdout()); +}); + +// See: https://github.com/BurntSushi/ripgrep/issues/210 +#[cfg(unix)] +rgtest!(r210, |dir: Dir, mut cmd: TestCommand| { + use std::ffi::OsStr; + use std::os::unix::ffi::OsStrExt; + + let badutf8 = OsStr::from_bytes(&b"foo\xffbar"[..]); + + // APFS does not support creating files with invalid UTF-8 bytes. + // https://github.com/BurntSushi/ripgrep/issues/559 + if dir.try_create(badutf8, "test").is_ok() { + cmd.arg("-H").arg("test").arg(badutf8); + assert_eq!(b"foo\xffbar:test\n".to_vec(), cmd.output().stdout); + } +}); + +// See: https://github.com/BurntSushi/ripgrep/issues/228 +rgtest!(r228, |dir: Dir, mut cmd: TestCommand| { + dir.create_dir("foo"); + + cmd.arg("--ignore-file").arg("foo").arg("test").assert_err(); +}); + +// See: https://github.com/BurntSushi/ripgrep/issues/229 +rgtest!(r229, |dir: Dir, mut cmd: TestCommand| { + dir.create("foo", "economie"); + + cmd.arg("-S").arg("[E]conomie").assert_err(); +}); + +// See: https://github.com/BurntSushi/ripgrep/issues/251 +rgtest!(r251, |dir: Dir, mut cmd: TestCommand| { + dir.create("foo", "привет\nПривет\nПрИвЕт"); + + let expected = "foo:привет\nfoo:Привет\nfoo:ПрИвЕт\n"; + eqnice!(expected, cmd.arg("-i").arg("привет").stdout()); +}); + +// See: https://github.com/BurntSushi/ripgrep/issues/256 +#[cfg(not(windows))] +rgtest!(r256, |dir: Dir, mut cmd: TestCommand| { + dir.create_dir("bar"); + dir.create("bar/baz", "test"); + dir.link_dir("bar", "foo"); + + eqnice!("foo/baz:test\n", cmd.arg("test").arg("foo").stdout()); +}); + +// See: https://github.com/BurntSushi/ripgrep/issues/256 +#[cfg(not(windows))] +rgtest!(r256_j1, |dir: Dir, mut cmd: TestCommand| { + dir.create_dir("bar"); + dir.create("bar/baz", "test"); + dir.link_dir("bar", "foo"); + + eqnice!("foo/baz:test\n", cmd.arg("-j1").arg("test").arg("foo").stdout()); +}); + +// See: https://github.com/BurntSushi/ripgrep/issues/270 +rgtest!(r270, |dir: Dir, mut cmd: TestCommand| { + dir.create("foo", "-test"); + + cmd.arg("-e").arg("-test"); + eqnice!("foo:-test\n", cmd.stdout()); +}); + +// See: https://github.com/BurntSushi/ripgrep/issues/279 +rgtest!(r279, |dir: Dir, mut cmd: TestCommand| { + dir.create("foo", "test"); + + eqnice!("", cmd.arg("-q").arg("test").stdout()); +}); + +// See: https://github.com/BurntSushi/ripgrep/issues/391 +rgtest!(r391, |dir: Dir, mut cmd: TestCommand| { + dir.create_dir(".git"); + dir.create("lock", ""); + dir.create("bar.py", ""); + dir.create(".git/packed-refs", ""); + dir.create(".git/description", ""); + + cmd.args(&[ + "--no-ignore", + "--hidden", + "--follow", + "--files", + "--glob", + "!{.git,node_modules,plugged}/**", + "--glob", + "*.{js,json,php,md,styl,scss,sass,pug,html,config,py,cpp,c,go,hs}", + ]); + eqnice!("bar.py\n", cmd.stdout()); +}); + +// See: https://github.com/BurntSushi/ripgrep/issues/405 +rgtest!(r405, |dir: Dir, mut cmd: TestCommand| { + dir.create_dir("foo/bar"); + dir.create_dir("bar/foo"); + dir.create("foo/bar/file1.txt", "test"); + dir.create("bar/foo/file2.txt", "test"); + + cmd.arg("-g").arg("!/foo/**").arg("test"); + eqnice!("bar/foo/file2.txt:test\n", cmd.stdout()); +}); + +// See: https://github.com/BurntSushi/ripgrep/issues/428 +#[cfg(not(windows))] +rgtest!(r428_color_context_path, |dir: Dir, mut cmd: TestCommand| { + dir.create("sherlock", "foo\nbar"); + cmd.args(&[ + "-A1", + "-H", + "--no-heading", + "-N", + "--colors=match:none", + "--color=always", + "--hyperlink-format=", + "foo", + ]); + + let expected = format!( + "{colored_path}:foo\n{colored_path}-bar\n", + colored_path = + "\x1b\x5b\x30\x6d\x1b\x5b\x33\x35\x6dsherlock\x1b\x5b\x30\x6d" + ); + eqnice!(expected, cmd.stdout()); +}); + +// See: https://github.com/BurntSushi/ripgrep/issues/428 +rgtest!(r428_unrecognized_style, |dir: Dir, mut cmd: TestCommand| { + dir.create("file.txt", "Sherlock"); + + cmd.arg("--colors=match:style:").arg("Sherlock"); + cmd.assert_err(); + + let output = cmd.raw_output(); + let stderr = String::from_utf8_lossy(&output.stderr); + let expected = "\ +rg: error parsing flag --colors: \ +unrecognized style attribute ''. Choose from: nobold, bold, nointense, \ +intense, nounderline, underline, noitalic, italic. +"; + eqnice!(expected, stderr); +}); + +// See: https://github.com/BurntSushi/ripgrep/issues/451 +rgtest!(r451_only_matching_as_in_issue, |dir: Dir, mut cmd: TestCommand| { + dir.create("digits.txt", "1 2 3\n"); + cmd.arg("--only-matching").arg(r"[0-9]+").arg("digits.txt"); + + let expected = "\ +1 +2 +3 +"; + eqnice!(expected, cmd.stdout()); +}); + +// See: https://github.com/BurntSushi/ripgrep/issues/451 +rgtest!(r451_only_matching, |dir: Dir, mut cmd: TestCommand| { + dir.create("digits.txt", "1 2 3\n123\n"); + cmd.args(&["--only-matching", "--column", r"[0-9]", "digits.txt"]); + + let expected = "\ +1:1:1 +1:3:2 +1:5:3 +2:1:1 +2:2:2 +2:3:3 +"; + eqnice!(expected, cmd.stdout()); +}); + +// See: https://github.com/BurntSushi/ripgrep/issues/483 +rgtest!(r483_matching_no_stdout, |dir: Dir, mut cmd: TestCommand| { + dir.create("file.py", ""); + cmd.arg("--quiet").arg("--files").arg("--glob").arg("*.py"); + eqnice!("", cmd.stdout()); +}); + +// See: https://github.com/BurntSushi/ripgrep/issues/483 +rgtest!(r483_non_matching_exit_code, |dir: Dir, mut cmd: TestCommand| { + dir.create("file.rs", ""); + cmd.arg("--quiet").arg("--files").arg("--glob").arg("*.py"); + cmd.assert_err(); +}); + +// See: https://github.com/BurntSushi/ripgrep/issues/493 +rgtest!(r493, |dir: Dir, mut cmd: TestCommand| { + dir.create("input.txt", "peshwaship 're seminomata"); + + cmd.arg("-o").arg(r"\b 're \b").arg("input.txt"); + assert_eq!(" 're \n", cmd.stdout()); +}); + +// See: https://github.com/BurntSushi/ripgrep/issues/506 +rgtest!(r506_word_not_parenthesized, |dir: Dir, mut cmd: TestCommand| { + dir.create("wb.txt", "min minimum amin\nmax maximum amax"); + cmd.arg("-w").arg("-o").arg("min|max").arg("wb.txt"); + eqnice!("min\nmax\n", cmd.stdout()); +}); + +// See: https://github.com/BurntSushi/ripgrep/issues/553 +rgtest!(r553_switch, |dir: Dir, mut cmd: TestCommand| { + dir.create("sherlock", SHERLOCK); + + let expected = "\ +sherlock:For the Doctor Watsons of this world, as opposed to the Sherlock +sherlock:be, to a very large extent, the result of luck. Sherlock Holmes +"; + cmd.arg("-i").arg("sherlock"); + eqnice!(expected, cmd.stdout()); + + // Repeat the `i` flag to make sure everything still works. + eqnice!(expected, cmd.arg("-i").stdout()); +}); + +rgtest!(r553_flag, |dir: Dir, mut cmd: TestCommand| { + dir.create("sherlock", SHERLOCK); + + let expected = "\ +For the Doctor Watsons of this world, as opposed to the Sherlock +Holmeses, success in the province of detective work must always +-- +but Doctor Watson has to have it taken out for him and dusted, +and exhibited clearly, with a label attached. +"; + cmd.arg("-C").arg("1").arg(r"world|attached").arg("sherlock"); + eqnice!(expected, cmd.stdout()); + + let expected = "\ +For the Doctor Watsons of this world, as opposed to the Sherlock +and exhibited clearly, with a label attached. +"; + eqnice!(expected, cmd.arg("-C").arg("0").stdout()); +}); + +// See: https://github.com/BurntSushi/ripgrep/issues/568 +rgtest!(r568_leading_hyphen_option_args, |dir: Dir, mut cmd: TestCommand| { + dir.create("file", "foo bar -baz\n"); + cmd.arg("-e-baz").arg("-e").arg("-baz").arg("file"); + eqnice!("foo bar -baz\n", cmd.stdout()); + + let mut cmd = dir.command(); + cmd.arg("-rni").arg("bar").arg("file"); + eqnice!("foo ni -baz\n", cmd.stdout()); + + let mut cmd = dir.command(); + cmd.arg("-r").arg("-n").arg("-i").arg("bar").arg("file"); + eqnice!("foo -n -baz\n", cmd.stdout()); +}); + +// See: https://github.com/BurntSushi/ripgrep/issues/599 +// +// This test used to check that we emitted color escape sequences even for +// empty matches, but with the addition of the JSON output format, clients no +// longer need to rely on escape sequences to parse matches. Therefore, we no +// longer emit useless escape sequences. +rgtest!(r599, |dir: Dir, mut cmd: TestCommand| { + dir.create("input.txt", "\n\ntest\n"); + cmd.args(&[ + "--color", + "ansi", + "--colors", + "path:none", + "--colors", + "line:none", + "--colors", + "match:fg:red", + "--colors", + "match:style:nobold", + "--line-number", + r"^$", + "input.txt", + ]); + + let expected = "\ +1: +2: +"; + eqnice_repr!(expected, cmd.stdout()); +}); + +// See: https://github.com/BurntSushi/ripgrep/issues/693 +rgtest!(r693_context_in_contextless_mode, |dir: Dir, mut cmd: TestCommand| { + dir.create("foo", "xyz\n"); + dir.create("bar", "xyz\n"); + + cmd.arg("-C1").arg("-c").arg("--sort-files").arg("xyz"); + eqnice!("bar:1\nfoo:1\n", cmd.stdout()); +}); + +// See: https://github.com/BurntSushi/ripgrep/issues/807 +rgtest!(r807, |dir: Dir, mut cmd: TestCommand| { + dir.create_dir(".git"); + dir.create(".gitignore", ".a/b"); + dir.create_dir(".a/b"); + dir.create_dir(".a/c"); + dir.create(".a/b/file", "test"); + dir.create(".a/c/file", "test"); + + eqnice!(".a/c/file:test\n", cmd.arg("--hidden").arg("test").stdout()); +}); + +// See: https://github.com/BurntSushi/ripgrep/pull/2711 +// +// Note that this isn't a regression test. In particular, this didn't fail +// with ripgrep 14.1.1. I couldn't figure out how to turn what the OP gave me +// into a failing test. +rgtest!(r2711, |dir: Dir, _cmd: TestCommand| { + dir.create_dir("a/b"); + dir.create("a/.ignore", ".foo"); + dir.create("a/b/.foo", ""); + + { + let mut cmd = dir.command(); + eqnice!("a/.ignore\n", cmd.arg("--hidden").arg("--files").stdout()); + } + { + let mut cmd = dir.command(); + eqnice!( + "./a/.ignore\n", + cmd.arg("--hidden").arg("--files").arg("./").stdout() + ); + } + + { + let mut cmd = dir.command(); + eqnice!( + "a/.ignore\n", + cmd.arg("--hidden").arg("--files").arg("a").stdout() + ); + } + { + let mut cmd = dir.command(); + cmd.arg("--hidden").arg("--files").arg("a/b").assert_err(); + } + { + let mut cmd = dir.command(); + eqnice!( + "./a/.ignore\n", + cmd.arg("--hidden").arg("--files").arg("./a").stdout() + ); + } + + { + let mut cmd = dir.command(); + cmd.current_dir("a"); + eqnice!(".ignore\n", cmd.arg("--hidden").arg("--files").stdout()); + } + { + let mut cmd = dir.command(); + cmd.current_dir("a/b"); + cmd.arg("--hidden").arg("--files").assert_err(); + } + { + let mut cmd = dir.command(); + cmd.current_dir("./a"); + eqnice!(".ignore\n", cmd.arg("--hidden").arg("--files").stdout()); + } +}); + +// See: https://github.com/BurntSushi/ripgrep/issues/829 +rgtest!(r829_original, |dir: Dir, _cmd: TestCommand| { + dir.create_dir("a/b"); + dir.create(".ignore", "/a/b"); + dir.create("a/b/test.txt", "Sample text"); + + { + let mut cmd = dir.command(); + cmd.args(&["Sample"]).assert_err(); + } + { + let mut cmd = dir.command(); + cmd.args(&["Sample", "a"]).assert_err(); + } + { + let mut cmd = dir.command(); + cmd.current_dir("a"); + cmd.args(&["Sample"]).assert_err(); + } +}); + +// See: https://github.com/BurntSushi/ripgrep/issues/2731 +rgtest!(r829_2731, |dir: Dir, _cmd: TestCommand| { + dir.create_dir("some_dir/build"); + dir.create("some_dir/build/foo", "string"); + dir.create(".ignore", "build/\n!/some_dir/build/"); + + { + let mut cmd = dir.command(); + eqnice!("some_dir/build/foo\n", cmd.arg("-l").arg("string").stdout()); + } + { + let mut cmd = dir.command(); + eqnice!( + "some_dir/build/foo\n", + cmd.arg("-l").arg("string").arg("some_dir").stdout() + ); + } + { + let mut cmd = dir.command(); + eqnice!( + "./some_dir/build/foo\n", + cmd.arg("-l").arg("string").arg("./some_dir").stdout() + ); + } + { + let mut cmd = dir.command(); + eqnice!( + "some_dir/build/foo\n", + cmd.arg("-l").arg("string").arg("some_dir/build").stdout() + ); + } + { + let mut cmd = dir.command(); + eqnice!( + "./some_dir/build/foo\n", + cmd.arg("-l").arg("string").arg("./some_dir/build").stdout() + ); + } +}); + +// See: https://github.com/BurntSushi/ripgrep/issues/2747 +rgtest!(r829_2747, |dir: Dir, _cmd: TestCommand| { + dir.create_dir("a/c/b"); + dir.create_dir("a/src/f/b"); + dir.create("a/c/b/foo", ""); + dir.create("a/src/f/b/foo", ""); + dir.create(".ignore", "/a/*/b"); + + { + let mut cmd = dir.command(); + eqnice!("a/src/f/b/foo\n", cmd.arg("--files").stdout()); + } + { + let mut cmd = dir.command(); + eqnice!("a/src/f/b/foo\n", cmd.arg("--files").arg("a/src").stdout()); + } + { + let mut cmd = dir.command(); + cmd.current_dir("a/src"); + eqnice!("f/b/foo\n", cmd.arg("--files").stdout()); + } +}); + +// See: https://github.com/BurntSushi/ripgrep/issues/2778 +rgtest!(r829_2778, |dir: Dir, _cmd: TestCommand| { + dir.create_dir("parent/subdir"); + dir.create(".ignore", "/parent/*.txt"); + dir.create("parent/ignore-me.txt", ""); + dir.create("parent/subdir/dont-ignore-me.txt", ""); + + { + let mut cmd = dir.command(); + eqnice!( + "parent/subdir/dont-ignore-me.txt\n", + cmd.arg("--files").stdout() + ); + } + { + let mut cmd = dir.command(); + cmd.current_dir("parent"); + eqnice!("subdir/dont-ignore-me.txt\n", cmd.arg("--files").stdout()); + } +}); + +// See: https://github.com/BurntSushi/ripgrep/issues/2836 +rgtest!(r829_2836, |dir: Dir, _cmd: TestCommand| { + dir.create_dir("testdir/sub/sub2"); + dir.create(".ignore", "/testdir/sub/sub2/\n"); + dir.create("testdir/sub/sub2/foo", ""); + + { + let mut cmd = dir.command(); + cmd.arg("--files").assert_err(); + } + { + let mut cmd = dir.command(); + cmd.current_dir("testdir"); + cmd.arg("--files").assert_err(); + } +}); + +// See: https://github.com/BurntSushi/ripgrep/pull/2933 +rgtest!(r829_2933, |dir: Dir, mut cmd: TestCommand| { + dir.create_dir("testdir/sub/sub2"); + dir.create(".ignore", "/testdir/sub/sub2/"); + dir.create("testdir/sub/sub2/testfile", "needle"); + + let args = &["--files-with-matches", "needle"]; + cmd.current_dir("testdir"); + cmd.args(args).assert_err(); +}); + +// See: https://github.com/BurntSushi/ripgrep/issues/900 +rgtest!(r900, |dir: Dir, mut cmd: TestCommand| { + dir.create("sherlock", SHERLOCK); + dir.create("pat", ""); + + cmd.arg("-fpat").arg("sherlock").assert_err(); +}); + +// See: https://github.com/BurntSushi/ripgrep/issues/1064 +rgtest!(r1064, |dir: Dir, mut cmd: TestCommand| { + dir.create("input", "abc"); + eqnice!("input:abc\n", cmd.arg("a(.*c)").stdout()); +}); + +// See: https://github.com/BurntSushi/ripgrep/issues/1174 +rgtest!(r1098, |dir: Dir, mut cmd: TestCommand| { + dir.create_dir(".git"); + dir.create(".gitignore", "a**b"); + dir.create("afoob", "test"); + cmd.arg("test").assert_err(); +}); + +// See: https://github.com/BurntSushi/ripgrep/issues/1130 +rgtest!(r1130, |dir: Dir, mut cmd: TestCommand| { + dir.create("foo", "test"); + eqnice!( + "foo\n", + cmd.arg("--files-with-matches").arg("test").arg("foo").stdout() + ); + + let mut cmd = dir.command(); + eqnice!( + "foo\n", + cmd.arg("--files-without-match").arg("nada").arg("foo").stdout() + ); +}); + +// See: https://github.com/BurntSushi/ripgrep/issues/1159 +rgtest!(r1159_invalid_flag, |_: Dir, mut cmd: TestCommand| { + cmd.arg("--wat").assert_exit_code(2); +}); + +// See: https://github.com/BurntSushi/ripgrep/issues/1159 +rgtest!(r1159_exit_status, |dir: Dir, _: TestCommand| { + dir.create("foo", "test"); + + // search with a match gets 0 exit status. + let mut cmd = dir.command(); + cmd.arg("test").assert_exit_code(0); + + // search with --quiet and a match gets 0 exit status. + let mut cmd = dir.command(); + cmd.arg("-q").arg("test").assert_exit_code(0); + + // search with a match and an error gets 2 exit status. + let mut cmd = dir.command(); + cmd.arg("test").arg("no-file").assert_exit_code(2); + + // search with a match in --quiet mode and an error gets 0 exit status. + let mut cmd = dir.command(); + cmd.arg("-q").arg("test").arg("foo").arg("no-file").assert_exit_code(0); + + // search with no match gets 1 exit status. + let mut cmd = dir.command(); + cmd.arg("nada").assert_exit_code(1); + + // search with --quiet and no match gets 1 exit status. + let mut cmd = dir.command(); + cmd.arg("-q").arg("nada").assert_exit_code(1); + + // search with no match and an error gets 2 exit status. + let mut cmd = dir.command(); + cmd.arg("nada").arg("no-file").assert_exit_code(2); + + // search with no match in --quiet mode and an error gets 2 exit status. + let mut cmd = dir.command(); + cmd.arg("-q").arg("nada").arg("foo").arg("no-file").assert_exit_code(2); +}); + +// See: https://github.com/BurntSushi/ripgrep/issues/1163 +rgtest!(r1163, |dir: Dir, mut cmd: TestCommand| { + dir.create("bom.txt", "\u{FEFF}test123\ntest123"); + eqnice!( + "bom.txt:test123\nbom.txt:test123\n", + cmd.arg("^test123").stdout() + ); +}); + +// See: https://github.com/BurntSushi/ripgrep/issues/1164 +rgtest!(r1164, |dir: Dir, mut cmd: TestCommand| { + dir.create_dir(".git"); + dir.create(".gitignore", "myfile"); + dir.create("MYFILE", "test"); + + cmd.arg("--ignore-file-case-insensitive").arg("test").assert_err(); + eqnice!( + "MYFILE:test\n", + cmd.arg("--no-ignore-file-case-insensitive").stdout() + ); +}); + +// See: https://github.com/BurntSushi/ripgrep/issues/1173 +rgtest!(r1173, |dir: Dir, mut cmd: TestCommand| { + dir.create_dir(".git"); + dir.create(".gitignore", "**"); + dir.create("foo", "test"); + cmd.arg("test").assert_err(); +}); + +// See: https://github.com/BurntSushi/ripgrep/issues/1174 +rgtest!(r1174, |dir: Dir, mut cmd: TestCommand| { + dir.create_dir(".git"); + dir.create(".gitignore", "**/**/*"); + dir.create_dir("a"); + dir.create("a/foo", "test"); + cmd.arg("test").assert_err(); +}); + +// See: https://github.com/BurntSushi/ripgrep/issues/1176 +rgtest!(r1176_literal_file, |dir: Dir, mut cmd: TestCommand| { + dir.create("patterns", "foo(bar\n"); + dir.create("test", "foo(bar"); + + eqnice!( + "foo(bar\n", + cmd.arg("-F").arg("-f").arg("patterns").arg("test").stdout() + ); +}); + +// See: https://github.com/BurntSushi/ripgrep/issues/1176 +rgtest!(r1176_line_regex, |dir: Dir, mut cmd: TestCommand| { + dir.create("patterns", "foo\n"); + dir.create("test", "foobar\nfoo\nbarfoo\n"); + + eqnice!( + "foo\n", + cmd.arg("-x").arg("-f").arg("patterns").arg("test").stdout() + ); +}); + +// See: https://github.com/BurntSushi/ripgrep/issues/1203 +rgtest!(r1203_reverse_suffix_literal, |dir: Dir, _: TestCommand| { + dir.create("test", "153.230000\n"); + + let mut cmd = dir.command(); + eqnice!("153.230000\n", cmd.arg(r"\d\d\d00").arg("test").stdout()); + + let mut cmd = dir.command(); + eqnice!("153.230000\n", cmd.arg(r"\d\d\d000").arg("test").stdout()); +}); + +// See: https://github.com/BurntSushi/ripgrep/issues/1223 +rgtest!( + r1223_no_dir_check_for_default_path, + |dir: Dir, mut cmd: TestCommand| { + dir.create_dir("-"); + dir.create("a.json", "{}"); + dir.create("a.txt", "some text"); + + eqnice!( + "a.json\na.txt\n", + sort_lines(&cmd.arg("a").pipe(b"a.json\na.txt")) + ); + } +); + +// See: https://github.com/BurntSushi/ripgrep/issues/1259 +rgtest!(r1259_drop_last_byte_nonl, |dir: Dir, mut cmd: TestCommand| { + dir.create("patterns-nonl", "[foo]"); + dir.create("patterns-nl", "[foo]\n"); + dir.create("test", "fz"); + + eqnice!("fz\n", cmd.arg("-f").arg("patterns-nonl").arg("test").stdout()); + cmd = dir.command(); + eqnice!("fz\n", cmd.arg("-f").arg("patterns-nl").arg("test").stdout()); +}); + +// See: https://github.com/BurntSushi/ripgrep/issues/1311 +rgtest!(r1311_multi_line_term_replace, |dir: Dir, mut cmd: TestCommand| { + dir.create("input", "hello\nworld\n"); + eqnice!( + "1:hello?world?\n", + cmd.args(&["-U", "-r?", "-n", "\n", "input"]).stdout() + ); +}); + +// See: https://github.com/BurntSushi/ripgrep/issues/1319 +rgtest!(r1319, |dir: Dir, mut cmd: TestCommand| { + dir.create("input", "CCAGCTACTCGGGAGGCTGAGGCTGGAGGATCGCTTGAGTCCAGGAGTTC"); + eqnice!( + "input:CCAGCTACTCGGGAGGCTGAGGCTGGAGGATCGCTTGAGTCCAGGAGTTC\n", + cmd.arg("TTGAGTCCAGGAG[ATCG]{2}C").stdout() + ); +}); + +// See: https://github.com/BurntSushi/ripgrep/issues/1332 +rgtest!(r1334_invert_empty_patterns, |dir: Dir, _cmd: TestCommand| { + dir.create("zero-patterns", ""); + dir.create("one-pattern", "\n"); + dir.create("haystack", "one\ntwo\nthree\n"); + + // zero patterns matches nothing + { + let mut cmd = dir.command(); + cmd.arg("-f").arg("zero-patterns").arg("haystack").assert_err(); + } + // one pattern that matches empty string matches everything + { + let mut cmd = dir.command(); + eqnice!( + "one\ntwo\nthree\n", + cmd.arg("-f").arg("one-pattern").arg("haystack").stdout() + ); + } + + // inverting zero patterns matches everything + // (This is the regression. ripgrep used to match nothing because of an + // incorrect optimization.) + { + let mut cmd = dir.command(); + eqnice!( + "one\ntwo\nthree\n", + cmd.arg("-vf").arg("zero-patterns").arg("haystack").stdout() + ); + } + // inverting one pattern that matches empty string matches nothing + { + let mut cmd = dir.command(); + cmd.arg("-vf").arg("one-pattern").arg("haystack").assert_err(); + } +}); + +// See: https://github.com/BurntSushi/ripgrep/issues/1334 +rgtest!(r1334_crazy_literals, |dir: Dir, mut cmd: TestCommand| { + dir.create("patterns", &"1.208.0.0/12\n".repeat(40)); + dir.create("corpus", "1.208.0.0/12\n"); + eqnice!( + "1.208.0.0/12\n", + cmd.arg("-Ff").arg("patterns").arg("corpus").stdout() + ); +}); + +// See: https://github.com/BurntSushi/ripgrep/issues/1380 +rgtest!(r1380, |dir: Dir, mut cmd: TestCommand| { + dir.create( + "foo", + "\ +a +b +c +d +e +d +e +d +e +d +e +", + ); + + eqnice!("d\ne\nd\n", cmd.args(&["-A2", "-m1", "d", "foo"]).stdout()); +}); + +// See: https://github.com/BurntSushi/ripgrep/issues/1389 +rgtest!(r1389_bad_symlinks_no_biscuit, |dir: Dir, mut cmd: TestCommand| { + dir.create_dir("mydir"); + dir.create("mydir/file.txt", "test"); + dir.link_dir("mydir", "mylink"); + + let stdout = cmd + .args(&["test", "--no-ignore", "--sort", "path", "mylink"]) + .stdout(); + eqnice!("mylink/file.txt:test\n", stdout); +}); + +// See: https://github.com/BurntSushi/ripgrep/issues/1401 +rgtest!(r1401_look_ahead_only_matching_1, |dir: Dir, mut cmd: TestCommand| { + // Only PCRE2 supports look-around. + if !dir.is_pcre2() { + return; + } + dir.create("ip.txt", "foo 42\nxoyz\ncat\tdog\n"); + cmd.args(&["-No", r".*o(?!.*\s)", "ip.txt"]); + eqnice!("xo\ncat\tdo\n", cmd.stdout()); + + let mut cmd = dir.command(); + cmd.args(&["-No", r".*o(?!.*[ \t])", "ip.txt"]); + eqnice!("xo\ncat\tdo\n", cmd.stdout()); +}); + +// See: https://github.com/BurntSushi/ripgrep/issues/1401 +rgtest!(r1401_look_ahead_only_matching_2, |dir: Dir, mut cmd: TestCommand| { + // Only PCRE2 supports look-around. + if !dir.is_pcre2() { + return; + } + dir.create("ip.txt", "foo 42\nxoyz\ncat\tdog\nfoo"); + cmd.args(&["-No", r".*o(?!.*\s)", "ip.txt"]); + eqnice!("xo\ncat\tdo\nfoo\n", cmd.stdout()); +}); + +// See: https://github.com/BurntSushi/ripgrep/issues/1412 +rgtest!(r1412_look_behind_no_replacement, |dir: Dir, mut cmd: TestCommand| { + // Only PCRE2 supports look-around. + if !dir.is_pcre2() { + return; + } + + dir.create("test", "foo\nbar\n"); + cmd.args(&["-nU", "-rquux", r"(?<=foo\n)bar", "test"]); + eqnice!("2:quux\n", cmd.stdout()); +}); + +// See: https://github.com/BurntSushi/ripgrep/pull/1446 +rgtest!( + r1446_respect_excludes_in_worktree, + |dir: Dir, mut cmd: TestCommand| { + dir.create_dir("repo/.git/info"); + dir.create("repo/.git/info/exclude", "ignored"); + dir.create_dir("repo/.git/worktrees/repotree"); + dir.create("repo/.git/worktrees/repotree/commondir", "../.."); + + dir.create_dir("repotree"); + dir.create("repotree/.git", "gitdir: repo/.git/worktrees/repotree"); + dir.create("repotree/ignored", ""); + dir.create("repotree/not-ignored", ""); + + cmd.arg("--sort").arg("path").arg("--files").arg("repotree"); + eqnice!("repotree/not-ignored\n", cmd.stdout()); + } +); + +// See: https://github.com/BurntSushi/ripgrep/issues/1537 +rgtest!(r1537, |dir: Dir, mut cmd: TestCommand| { + dir.create("foo", "abc;de,fg"); + + let expected = "foo:abc;de,fg\n"; + eqnice!(expected, cmd.arg(";(.*,){1}").stdout()); +}); + +// See: https://github.com/BurntSushi/ripgrep/issues/1559 +rgtest!(r1559, |dir: Dir, mut cmd: TestCommand| { + dir.create( + "foo", + "\ +type A struct { + TaskID int `json:\"taskID\"` +} + +type B struct { + ObjectID string `json:\"objectID\"` + TaskID int `json:\"taskID\"` +} +", + ); + + let expected = "\ +foo: TaskID int `json:\"taskID\"` +foo: TaskID int `json:\"taskID\"` +"; + eqnice!(expected, cmd.arg("TaskID +int").stdout()); +}); + +// See: https://github.com/BurntSushi/ripgrep/issues/1573 +// +// Tests that if look-ahead is used, then --count-matches is correct. +rgtest!(r1573, |dir: Dir, mut cmd: TestCommand| { + // Only PCRE2 supports look-ahead. + if !dir.is_pcre2() { + return; + } + + dir.create_bytes("foo", b"\xFF\xFE\x00\x62"); + dir.create( + "foo", + "\ +def A; +def B; +use A; +use B; +", + ); + + // Check that normal --count is correct. + cmd.args(&[ + "--pcre2", + "--multiline", + "--count", + r"(?s)def (\w+);(?=.*use \w+)", + "foo", + ]); + eqnice!("2\n", cmd.stdout()); + + // Now check --count-matches. + let mut cmd = dir.command(); + cmd.args(&[ + "--pcre2", + "--multiline", + "--count-matches", + r"(?s)def (\w+);(?=.*use \w+)", + "foo", + ]); + eqnice!("2\n", cmd.stdout()); +}); + +// See: https://github.com/BurntSushi/ripgrep/issues/1638 +// +// Tests if UTF-8 BOM is sniffed, then the column index is correct. +rgtest!(r1638, |dir: Dir, mut cmd: TestCommand| { + dir.create_bytes("foo", b"\xef\xbb\xbfx"); + + eqnice!("foo:1:1:x\n", cmd.arg("--column").arg("x").stdout()); +}); + +// See: https://github.com/BurntSushi/ripgrep/issues/1739 +rgtest!(r1739_replacement_lineterm_match, |dir: Dir, mut cmd: TestCommand| { + dir.create("test", "a\n"); + cmd.args(&[r"-r${0}f", r".*", "test"]); + eqnice!("af\n", cmd.stdout()); +}); + +// See: https://github.com/BurntSushi/ripgrep/issues/1757 +rgtest!(f1757, |dir: Dir, _: TestCommand| { + dir.create_dir("rust/target"); + dir.create(".ignore", "rust/target"); + dir.create("rust/source.rs", "needle"); + dir.create("rust/target/rustdoc-output.html", "needle"); + + let args = &["--files-with-matches", "needle", "rust"]; + eqnice!("rust/source.rs\n", dir.command().args(args).stdout()); + let args = &["--files-with-matches", "needle", "./rust"]; + eqnice!("./rust/source.rs\n", dir.command().args(args).stdout()); + + dir.create_dir("rust1/target/onemore"); + dir.create(".ignore", "rust1/target/onemore"); + dir.create("rust1/source.rs", "needle"); + dir.create("rust1/target/onemore/rustdoc-output.html", "needle"); + let args = &["--files-with-matches", "needle", "rust1"]; + eqnice!("rust1/source.rs\n", dir.command().args(args).stdout()); + let args = &["--files-with-matches", "needle", "./rust1"]; + eqnice!("./rust1/source.rs\n", dir.command().args(args).stdout()); +}); + +// See: https://github.com/BurntSushi/ripgrep/issues/1765 +rgtest!(r1765, |dir: Dir, mut cmd: TestCommand| { + dir.create("test", "\n"); + // We need to add --color=always here to force the failure, since the bad + // code path is only triggered when colors are enabled. + cmd.args(&[r"x?", "--crlf", "--color", "always"]); + + assert!(!cmd.stdout().is_empty()); +}); + +// See: https://github.com/BurntSushi/ripgrep/issues/1838 +rgtest!(r1838_nul_error_with_binary_detection, |dir: Dir, _: TestCommand| { + // We don't support this error reporting with PCRE2 since we can't parse + // the pattern (easily) to give a good error message. + if dir.is_pcre2() { + return; + } + dir.create("test", "foo\n"); + + dir.command().args(&[r"foo\x00?"]).assert_err(); + eqnice!("test:foo\n", dir.command().args(&["-a", r"foo\x00?"]).stdout()); +}); + +// See: https://github.com/BurntSushi/ripgrep/issues/1866 +rgtest!(r1866, |dir: Dir, mut cmd: TestCommand| { + dir.create("test", "foobar\nfoobar\nfoo quux"); + cmd.args(&[ + "--multiline", + "--vimgrep", + r"foobar\nfoobar\nfoo|quux", + "test", + ]); + + // vimgrep only wants the first line of each match, even when a match + // spans multiple lines. + // + // See: https://github.com/BurntSushi/ripgrep/issues/1866 + let expected = "\ +test:1:1:foobar +test:3:5:foo quux +"; + eqnice!(expected, cmd.stdout()); +}); + +// See: https://github.com/BurntSushi/ripgrep/issues/1868 +rgtest!(r1868_context_passthru_override, |dir: Dir, _: TestCommand| { + dir.create("test", "foo\nbar\nbaz\nquux\n"); + + let args = &["-C1", "bar", "test"]; + eqnice!("foo\nbar\nbaz\n", dir.command().args(args).stdout()); + let args = &["--passthru", "bar", "test"]; + eqnice!("foo\nbar\nbaz\nquux\n", dir.command().args(args).stdout()); + + let args = &["--passthru", "-C1", "bar", "test"]; + eqnice!("foo\nbar\nbaz\n", dir.command().args(args).stdout()); + let args = &["-C1", "--passthru", "bar", "test"]; + eqnice!("foo\nbar\nbaz\nquux\n", dir.command().args(args).stdout()); + + let args = &["--passthru", "-B1", "bar", "test"]; + eqnice!("foo\nbar\n", dir.command().args(args).stdout()); + let args = &["-B1", "--passthru", "bar", "test"]; + eqnice!("foo\nbar\nbaz\nquux\n", dir.command().args(args).stdout()); + + let args = &["--passthru", "-A1", "bar", "test"]; + eqnice!("bar\nbaz\n", dir.command().args(args).stdout()); + let args = &["-A1", "--passthru", "bar", "test"]; + eqnice!("foo\nbar\nbaz\nquux\n", dir.command().args(args).stdout()); +}); + +rgtest!(r1878, |dir: Dir, _: TestCommand| { + dir.create("test", "a\nbaz\nabc\n"); + + // Since ripgrep enables (?m) by default, '^' will match at the beginning + // of a line, even when -U/--multiline is used. + let args = &["-U", "--no-mmap", r"^baz", "test"]; + eqnice!("baz\n", dir.command().args(args).stdout()); + let args = &["-U", "--mmap", r"^baz", "test"]; + eqnice!("baz\n", dir.command().args(args).stdout()); + + // But when (?-m) is disabled, or when \A is used, then there should be no + // matches that aren't anchored to the beginning of the file. + let args = &["-U", "--no-mmap", r"(?-m)^baz", "test"]; + dir.command().args(args).assert_err(); + let args = &["-U", "--mmap", r"(?-m)^baz", "test"]; + dir.command().args(args).assert_err(); + + let args = &["-U", "--no-mmap", r"\Abaz", "test"]; + dir.command().args(args).assert_err(); + let args = &["-U", "--mmap", r"\Abaz", "test"]; + dir.command().args(args).assert_err(); +}); + +// See: https://github.com/BurntSushi/ripgrep/issues/1891 +rgtest!(r1891, |dir: Dir, mut cmd: TestCommand| { + dir.create("test", "\n##\n"); + // N.B. We use -o here to force the issue to occur, which seems to only + // happen when each match needs to be detected. + eqnice!("1:\n2:\n2:\n2:\n", cmd.args(&["-won", "", "test"]).stdout()); +}); + +// See: https://github.com/BurntSushi/ripgrep/issues/2094 +rgtest!(r2094, |dir: Dir, mut cmd: TestCommand| { + dir.create("haystack", "a\nb\nc\na\nb\nc"); + cmd.args(&[ + "--no-line-number", + "--no-filename", + "--multiline", + "--max-count=1", + "--passthru", + "--replace=B", + "b", + "haystack", + ]); + let expected = "\ +a +B +c +a +b +c +"; + eqnice!(expected, cmd.stdout()); +}); + +// See: https://github.com/BurntSushi/ripgrep/issues/2095 +rgtest!(r2095, |dir: Dir, mut cmd: TestCommand| { + dir.create( + "test", + "#!/usr/bin/env bash + +zero=one + +a=one + +if true; then + a=( + a + b + c + ) + true +fi + +a=two + +b=one +}); +", + ); + cmd.args(&[ + "--line-number", + "--multiline", + "--only-matching", + "--replace", + "${value}", + r"^(?P\s*)a=(?P(?ms:[(].*?[)])|.*?)$", + "test", + ]); + let expected = "4:one +8:( +9: a +10: b +11: c +12: ) +15:two +"; + eqnice!(expected, cmd.stdout()); +}); + +// See: https://github.com/BurntSushi/ripgrep/issues/2198 +rgtest!(r2198, |dir: Dir, mut cmd: TestCommand| { + dir.create(".ignore", "a"); + dir.create(".rgignore", "b"); + dir.create("a", ""); + dir.create("b", ""); + dir.create("c", ""); + + cmd.arg("--files").arg("--sort").arg("path"); + eqnice!("c\n", cmd.stdout()); + eqnice!("a\nb\nc\n", cmd.arg("--no-ignore-dot").stdout()); +}); + +// See: https://github.com/BurntSushi/ripgrep/issues/2208 +rgtest!(r2208, |dir: Dir, mut cmd: TestCommand| { + dir.create("test", "# Compile requirements.txt files from all found or specified requirements.in files (compile). +# Use -h to include hashes, -u dep1,dep2... to upgrade specific dependencies, and -U to upgrade all. +pipc () { # [-h] [-U|-u [,...]] [...] [-- ...] + emulate -L zsh + unset REPLY + if [[ $1 == --help ]] { zpy $0; return } + [[ $ZPY_PROCS ]] || return + + local gen_hashes upgrade upgrade_csv + while [[ $1 == -[hUu] ]] { + if [[ $1 == -h ]] { gen_hashes=--generate-hashes; shift } + if [[ $1 == -U ]] { upgrade=1; shift } + if [[ $1 == -u ]] { upgrade=1; upgrade_csv=$2; shift 2 } + } +} +"); + cmd.args(&[ + "-N", + "-U", + "-r", "$usage", + r#"^(?P\n?(# .*\n)*)(alias (?Ppipc)="[^"]+"|(?Ppipc) \(\) \{)( #(?P .+))?"#, + "test", + ]); + let expected = " [-h] [-U|-u [,...]] [...] [-- ...]\n"; + eqnice!(expected, cmd.stdout()); +}); + +// See: https://github.com/BurntSushi/ripgrep/issues/2236 +rgtest!(r2236, |dir: Dir, mut cmd: TestCommand| { + dir.create(".ignore", r"foo\/"); + dir.create_dir("foo"); + dir.create("foo/bar", "test\n"); + cmd.args(&["test"]).assert_err(); +}); + +// See: https://github.com/BurntSushi/ripgrep/issues/2480 +rgtest!(r2480, |dir: Dir, mut cmd: TestCommand| { + dir.create("file", "FooBar\n"); + + // no regression in empty pattern behavior + cmd.args(&["-e", "", "file"]); + eqnice!("FooBar\n", cmd.stdout()); + + // no regression in single pattern behavior + let mut cmd = dir.command(); + cmd.args(&["-e", ")(", "file"]); + eqnice!("FooBar\n", cmd.stdout()); + + // no regression in multiple patterns behavior + let mut cmd = dir.command(); + cmd.args(&["--only-matching", "-e", "Foo", "-e", "Bar", "file"]); + eqnice!("Foo\nBar\n", cmd.stdout()); + + // no regression in capture groups behavior + let mut cmd = dir.command(); + cmd.args(&["-e", "Fo(oB)a(r)", "--replace", "${0}_${1}_${2}${3}", "file"]); + eqnice!("FooBar_oB_r\n", cmd.stdout()); // note: ${3} expected to be empty + + // flag does not leak into next pattern on match + let mut cmd = dir.command(); + cmd.args(&["--only-matching", "-e", "(?i)foo", "-e", "bar", "file"]); + eqnice!("Foo\n", cmd.stdout()); + + // flag does not leak into next pattern on mismatch + let mut cmd = dir.command(); + cmd.args(&["--only-matching", "-e", "(?i)notfoo", "-e", "bar", "file"]); + cmd.assert_err(); +}); + +// See: https://github.com/BurntSushi/ripgrep/issues/2574 +rgtest!(r2574, |dir: Dir, mut cmd: TestCommand| { + dir.create("haystack", "some.domain.com\nsome.domain.com/x\n"); + let got = cmd + .args(&[ + "--no-filename", + "--no-unicode", + "-w", + "-o", + r"(\w+\.)*domain\.(\w+)", + ]) + .stdout(); + eqnice!("some.domain.com\nsome.domain.com\n", got); +}); + +// See: https://github.com/BurntSushi/ripgrep/issues/2658 +rgtest!(r2658_null_data_line_regexp, |dir: Dir, mut cmd: TestCommand| { + dir.create("haystack", "foo\0bar\0quux\0"); + let got = cmd.args(&["--null-data", "--line-regexp", r"bar"]).stdout(); + eqnice!("haystack:bar\0", got); +}); + +// See: https://github.com/BurntSushi/ripgrep/issues/2770 +rgtest!(r2770_gitignore_error, |dir: Dir, _cmd: TestCommand| { + dir.create(".git", ""); + dir.create(".gitignore", "**/bar/*"); + dir.create_dir("foo/bar"); + dir.create("foo/bar/baz", "quux"); + + dir.command().arg("-l").arg("quux").assert_err(); + dir.command().current_dir("foo").arg("-l").arg("quux").assert_err(); +}); + +// See: https://github.com/BurntSushi/ripgrep/pull/2944 +rgtest!(r2944_incorrect_bytes_searched, |dir: Dir, mut cmd: TestCommand| { + dir.create("haystack", "foo1\nfoo2\nfoo3\nfoo4\nfoo5\n"); + let got = cmd.args(&["--stats", "-m2", "foo", "."]).stdout(); + assert!(got.contains("10 bytes searched\n")); +}); + +// See: https://github.com/BurntSushi/ripgrep/issues/2990 +#[cfg(unix)] +rgtest!(r2990_trip_over_trailing_dot, |dir: Dir, _cmd: TestCommand| { + dir.create_dir("asdf"); + dir.create_dir("asdf."); + dir.create("asdf/foo", ""); + dir.create("asdf./foo", ""); + + let got = dir.command().args(&["--files", "-g", "!asdf/"]).stdout(); + eqnice!("asdf./foo\n", got); + + // This used to ignore the glob given and included `asdf./foo` in output. + let got = dir.command().args(&["--files", "-g", "!asdf./"]).stdout(); + eqnice!("asdf/foo\n", got); +}); + +// See: https://github.com/BurntSushi/ripgrep/issues/3067 +rgtest!(r3067_gitignore_error, |dir: Dir, mut cmd: TestCommand| { + dir.create(".git", ""); + dir.create(".gitignore", "foobar/debug"); + dir.create_dir("foobar/some/debug"); + dir.create_dir("foobar/debug"); + dir.create("foobar/some/debug/flag", "baz"); + dir.create("foobar/debug/flag2", "baz"); + + let got = cmd.arg("baz").stdout(); + eqnice!("foobar/some/debug/flag:baz\n", got); +}); + +// See: https://github.com/BurntSushi/ripgrep/issues/3108 +rgtest!(r3108_files_without_match_quiet_exit, |dir: Dir, _: TestCommand| { + dir.create("yes-match", "abc"); + dir.create("non-match", "xyz"); + + dir.command().args(&["-q", "abc", "non-match"]).assert_exit_code(1); + dir.command().args(&["-q", "abc", "yes-match"]).assert_exit_code(0); + dir.command() + .args(&["--files-with-matches", "-q", "abc", "non-match"]) + .assert_exit_code(1); + dir.command() + .args(&["--files-with-matches", "-q", "abc", "yes-match"]) + .assert_exit_code(0); + + dir.command() + .args(&["--files-without-match", "abc", "non-match"]) + .assert_exit_code(0); + dir.command() + .args(&["--files-without-match", "abc", "yes-match"]) + .assert_exit_code(1); + + let got = dir + .command() + .args(&["--files-without-match", "abc", "non-match"]) + .stdout(); + eqnice!("non-match\n", got); + + dir.command() + .args(&["--files-without-match", "-q", "abc", "non-match"]) + .assert_exit_code(0); + dir.command() + .args(&["--files-without-match", "-q", "abc", "yes-match"]) + .assert_exit_code(1); + + let got = dir + .command() + .args(&["--files-without-match", "-q", "abc", "non-match"]) + .stdout(); + eqnice!("", got); +}); + +// See: https://github.com/BurntSushi/ripgrep/issues/3127 +rgtest!( + r3127_gitignore_allow_unclosed_class, + |dir: Dir, mut cmd: TestCommand| { + dir.create_dir(".git"); + dir.create(".gitignore", "[abc"); + dir.create("[abc", ""); + dir.create("test", ""); + + let got = cmd.args(&["--files"]).stdout(); + eqnice!("test\n", got); + } +); + +// See: https://github.com/BurntSushi/ripgrep/issues/3127 +rgtest!( + r3127_glob_flag_not_allow_unclosed_class, + |dir: Dir, mut cmd: TestCommand| { + dir.create("[abc", ""); + dir.create("test", ""); + + cmd.args(&["--files", "-g", "[abc"]).assert_err(); + } +); + +// See: https://github.com/BurntSushi/ripgrep/issues/3139 +rgtest!( + r3139_multiline_lookahead_files_with_matches, + |dir: Dir, _cmd: TestCommand| { + // Only PCRE2 supports look-around. + if !dir.is_pcre2() { + return; + } + dir.create( + "test", + "\ +Start \n \n\n \ +XXXXXXXXXXXXXXXXXXXXXXXXXX\n \ +YYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYYY\n \ +\n thing2 \n\n", + ); + + let got = dir + .command() + .args(&[ + "--multiline", + "--pcre2", + r"(?s)Start(?=.*thing2)", + "test", + ]) + .stdout(); + eqnice!("Start \n", got); + + let got = dir + .command() + .args(&[ + "--multiline", + "--pcre2", + "--files-with-matches", + r"(?s)Start(?=.*thing2)", + "test", + ]) + .stdout(); + eqnice!("test\n", got); + } +); + +// See: https://github.com/BurntSushi/ripgrep/issues/3173 +rgtest!(r3173_hidden_whitelist_only_dot, |dir: Dir, _: TestCommand| { + dir.create_dir("subdir"); + dir.create("subdir/.foo.txt", "text"); + dir.create(".ignore", "!.foo.txt"); + + let cmd = || dir.command(); + eqnice!(cmd().args(&["--files"]).stdout(), "subdir/.foo.txt\n"); + eqnice!(cmd().args(&["--files", "."]).stdout(), "./subdir/.foo.txt\n"); + eqnice!(cmd().args(&["--files", "./"]).stdout(), "./subdir/.foo.txt\n"); + + let cmd = || { + let mut cmd = dir.command(); + cmd.current_dir("subdir"); + cmd + }; + eqnice!(cmd().args(&["--files"]).stdout(), ".foo.txt\n"); + eqnice!(cmd().args(&["--files", "."]).stdout(), "./.foo.txt\n"); + eqnice!(cmd().args(&["--files", "./"]).stdout(), "./.foo.txt\n"); +}); + +// See: https://github.com/BurntSushi/ripgrep/issues/3179 +rgtest!(r3179_global_gitignore_cwd, |dir: Dir, mut cmd: TestCommand| { + dir.create_dir("a/b/c"); + dir.create("a/b/c/haystack", ""); + dir.create(".test.gitignore", "/haystack"); + + // I'm not sure in which cases this can fail. If it + // does and it's unavoidable, feel free to submit a + // patch that skips this test when this canonicalization + // fails. + // + // The reason we canonicalize here is strange, and it is + // perhaps papering over a bug in ripgrep. But on macOS, + // `TMPDIR` is set to `/var/blah/blah`. However, `/var` + // is symlinked to `/private/var`. So the CWD detected by + // the process is `/private/var`. So it turns out that the + // CWD is not a proper prefix of `dir.path()` here. So we + // cheat around this by forcing our path to be canonicalized + // so it's `/private/var` everywhere. + // + // Arguably, ripgrep should still work here without + // canonicalization. But it's not actually quite clear + // to me how to do it. I *believe* the solution here is + // that gitignore matching should be relative to the directory + // path given to `WalkBuilder::{add,new}`, and *not* to the + // CWD. But this is a very big change to how `ignore` works + // I think. At least conceptually. So that will need to be + // something we do when we rewrite `ignore`. Sigh. + // + // ... but, on Windows, path canonicalization seems to + // totally fuck things up, so skip it there. HEAVY sigh. + let dir_path = if cfg!(windows) { + dir.path().to_path_buf() + } else { + dir.path().canonicalize().unwrap() + }; + let ignore_file_path = dir_path.join(".test.gitignore"); + cmd.current_dir("a/b/c") + .arg("--files") + .arg("--ignore-file") + .arg(ignore_file_path.display().to_string()) + // This is a key part of the reproduction. When just providing `.` + // to ignore's walker (as ripgrep does when a path to search isn't + // provided), then everything works as one expects. Because there's + // nothing to strip off of the paths being searched. But when one + // provides an absolute path, the stripping didn't work. + .arg(&dir_path) + .assert_err(); +}); + +// See: https://github.com/BurntSushi/ripgrep/issues/3180 +rgtest!(r3180_look_around_panic, |dir: Dir, mut cmd: TestCommand| { + dir.create("haystack", " b b b b b b b b\nc\n"); + + let got = cmd + .arg(r#"(^|[^a-z])((([a-z]+)?)\s)?b(\s([a-z]+)?)($|[^a-z])"#) + .arg("haystack") + .arg("-U") + .arg("-rx") + .stdout(); + eqnice!("xbxbx\n", got); +}); + +// See: https://github.com/BurntSushi/ripgrep/issues/3275 +// +// This test doesn't seem to work on Windows for whatever reason. I haven't +// investigated it yet, but I'm pretty sure it's just the test. I'm guessing +// it's related to path separators. ---AG +#[cfg(not(windows))] +rgtest!(r3275_git_global_config_env, |dir: Dir, mut cmd: TestCommand| { + dir.create_dir("foo"); + dir.create(".git", ""); + dir.create("foo/foo1", ""); + dir.create("foo/foo2", ""); + dir.create("global-excludes-nonstandard", "foo2"); + dir.create( + "global-config-nonstandard", + &format!( + "[core]\n\texcludesFile = {path}", + path = dir.path().join("global-excludes-nonstandard").display() + ), + ); + + let config_path = dir.path().join("global-config-nonstandard"); + cmd.env("GIT_CONFIG_GLOBAL", config_path); + eqnice!("foo/foo1\n", cmd.arg("--files").arg("foo").stdout()); +}); diff --git a/userland/ripgrep/tests/tests.rs b/userland/ripgrep/tests/tests.rs new file mode 100644 index 0000000..81e40f8 --- /dev/null +++ b/userland/ripgrep/tests/tests.rs @@ -0,0 +1,22 @@ +// Macros useful for testing. +#[macro_use] +mod macros; + +// Corpora. +mod hay; +// Utilities for making tests nicer to read and easier to write. +mod util; + +// Tests for ripgrep's handling of binary files. +mod binary; +// Tests related to most features in ripgrep. If you're adding something new +// to ripgrep, tests should probably go in here. +mod feature; +// Tests for ripgrep's JSON format. +mod json; +// Miscellaneous tests grouped in a haphazard manner. Try not to add more. +mod misc; +// Tests for ripgrep's multiline search support. +mod multiline; +// Regression tests. +mod regression; diff --git a/userland/ripgrep/tests/util.rs b/userland/ripgrep/tests/util.rs new file mode 100644 index 0000000..db7331c --- /dev/null +++ b/userland/ripgrep/tests/util.rs @@ -0,0 +1,524 @@ +use std::env; +use std::error; +use std::ffi::OsStr; +use std::fs::{self, File}; +use std::io::{self, Write}; +use std::path::{Path, PathBuf}; +use std::process::{self, Command}; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::thread; +use std::time::Duration; + +use bstr::ByteSlice; + +static TEST_DIR: &'static str = "ripgrep-tests"; +static NEXT_ID: AtomicUsize = AtomicUsize::new(0); + +/// Setup an empty work directory and return a command pointing to the ripgrep +/// executable whose CWD is set to the work directory. +/// +/// The name given will be used to create the directory. Generally, it should +/// correspond to the test name. +pub fn setup(test_name: &str) -> (Dir, TestCommand) { + let dir = Dir::new(test_name); + let cmd = dir.command(); + (dir, cmd) +} + +/// Like `setup`, but uses PCRE2 as the underlying regex engine. +pub fn setup_pcre2(test_name: &str) -> (Dir, TestCommand) { + let mut dir = Dir::new(test_name); + dir.pcre2(true); + let cmd = dir.command(); + (dir, cmd) +} + +/// Break the given string into lines, sort them and then join them back +/// together. This is useful for testing output from ripgrep that may not +/// always be in the same order. +pub fn sort_lines(lines: &str) -> String { + let mut lines: Vec<&str> = lines.trim().lines().collect(); + lines.sort(); + format!("{}\n", lines.join("\n")) +} + +/// Returns true if and only if the given program can be successfully executed +/// with a `--help` flag. +pub fn cmd_exists(program: &str) -> bool { + match Command::new(program).arg("--help").output() { + Ok(output) => output.status.success(), + Err(_) => false, + } +} + +/// Dir represents a directory in which tests should be run. +/// +/// Directories are created from a global atomic counter to avoid duplicates. +#[derive(Clone, Debug)] +pub struct Dir { + /// The directory in which this test executable is running. + root: PathBuf, + /// The directory in which the test should run. If a test needs to create + /// files, they should go in here. This directory is also used as the CWD + /// for any processes created by the test. + dir: PathBuf, + /// Set to true when the test should use PCRE2 as the regex engine. + pcre2: bool, +} + +impl Dir { + /// Create a new test working directory with the given name. The name + /// does not need to be distinct for each invocation, but should correspond + /// to a logical grouping of tests. + pub fn new(name: &str) -> Dir { + let id = NEXT_ID.fetch_add(1, Ordering::Relaxed); + let root = env::current_exe() + .unwrap() + .parent() + .expect("executable's directory") + .to_path_buf(); + let dir = + env::temp_dir().join(TEST_DIR).join(name).join(&format!("{id}")); + if dir.exists() { + nice_err(&dir, fs::remove_dir_all(&dir)); + } + nice_err(&dir, repeat(|| fs::create_dir_all(&dir))); + Dir { root, dir, pcre2: false } + } + + /// Use PCRE2 for this test. + pub fn pcre2(&mut self, yes: bool) { + self.pcre2 = yes; + } + + /// Returns true if and only if this test is configured to use PCRE2 as + /// the regex engine. + pub fn is_pcre2(&self) -> bool { + self.pcre2 + } + + /// Create a new file with the given name and contents in this directory, + /// or panic on error. + pub fn create>(&self, name: P, contents: &str) { + self.create_bytes(name, contents.as_bytes()); + } + + /// Try to create a new file with the given name and contents in this + /// directory. + #[allow(dead_code)] // unused on Windows + pub fn try_create>( + &self, + name: P, + contents: &str, + ) -> io::Result<()> { + let path = self.dir.join(name); + self.try_create_bytes(path, contents.as_bytes()) + } + + /// Create a new file with the given name and size. + pub fn create_size>(&self, name: P, filesize: u64) { + let path = self.dir.join(name); + let file = nice_err(&path, File::create(&path)); + nice_err(&path, file.set_len(filesize)); + } + + /// Create a new file with the given name and contents in this directory, + /// or panic on error. + pub fn create_bytes>(&self, name: P, contents: &[u8]) { + let path = self.dir.join(&name); + nice_err(&path, self.try_create_bytes(name, contents)); + } + + /// Try to create a new file with the given name and contents in this + /// directory. + pub fn try_create_bytes>( + &self, + name: P, + contents: &[u8], + ) -> io::Result<()> { + let path = self.dir.join(name); + let mut file = File::create(path)?; + file.write_all(contents)?; + file.flush() + } + + /// Remove a file with the given name from this directory. + pub fn remove>(&self, name: P) { + let path = self.dir.join(name); + nice_err(&path, fs::remove_file(&path)); + } + + /// Create a new directory with the given path (and any directories above + /// it) inside this directory. + pub fn create_dir>(&self, path: P) { + let path = self.dir.join(path); + nice_err(&path, repeat(|| fs::create_dir_all(&path))); + } + + /// Creates a new command that is set to use the ripgrep executable in + /// this working directory. + /// + /// This also: + /// + /// * Unsets the `RIPGREP_CONFIG_PATH` environment variable. + /// * Sets the `--path-separator` to `/` so that paths have the same output + /// on all systems. Tests that need to check `--path-separator` itself + /// can simply pass it again to override it. + pub fn command(&self) -> TestCommand { + let mut cmd = self.bin(); + cmd.env_remove("RIPGREP_CONFIG_PATH"); + cmd.current_dir(&self.dir); + cmd.arg("--path-separator").arg("/"); + if self.is_pcre2() { + cmd.arg("--pcre2"); + } + TestCommand { dir: self.clone(), cmd } + } + + /// Returns the path to the ripgrep executable. + pub fn bin(&self) -> process::Command { + let rg = self.root.join(format!("../rg{}", env::consts::EXE_SUFFIX)); + match cross_runner() { + None => process::Command::new(rg), + Some(runner) => { + let mut cmd = process::Command::new(runner); + cmd.arg(rg); + cmd + } + } + } + + /// Returns the path to this directory. + pub fn path(&self) -> &Path { + &self.dir + } + + /// Creates a directory symlink to the src with the given target name + /// in this directory. + #[cfg(not(windows))] + pub fn link_dir, T: AsRef>(&self, src: S, target: T) { + use std::os::unix::fs::symlink; + let src = self.dir.join(src); + let target = self.dir.join(target); + let _ = fs::remove_file(&target); + nice_err(&target, symlink(&src, &target)); + } + + /// Creates a directory symlink to the src with the given target name + /// in this directory. + #[cfg(windows)] + pub fn link_dir, T: AsRef>(&self, src: S, target: T) { + use std::os::windows::fs::symlink_dir; + let src = self.dir.join(src); + let target = self.dir.join(target); + let _ = fs::remove_dir(&target); + nice_err(&target, symlink_dir(&src, &target)); + } + + /// Creates a file symlink to the src with the given target name + /// in this directory. + #[cfg(not(windows))] + pub fn link_file, T: AsRef>( + &self, + src: S, + target: T, + ) { + self.link_dir(src, target); + } + + /// Creates a file symlink to the src with the given target name + /// in this directory. + #[cfg(windows)] + #[allow(dead_code)] // unused on Windows + pub fn link_file, T: AsRef>( + &self, + src: S, + target: T, + ) { + use std::os::windows::fs::symlink_file; + let src = self.dir.join(src); + let target = self.dir.join(target); + let _ = fs::remove_file(&target); + nice_err(&target, symlink_file(&src, &target)); + } +} + +/// A simple wrapper around a process::Command with some conveniences. +#[derive(Debug)] +pub struct TestCommand { + /// The dir used to launched this command. + dir: Dir, + /// The actual command we use to control the process. + cmd: Command, +} + +impl TestCommand { + /// Returns a mutable reference to the underlying command. + pub fn cmd(&mut self) -> &mut Command { + &mut self.cmd + } + + /// Add an argument to pass to the command. + pub fn arg>(&mut self, arg: A) -> &mut TestCommand { + self.cmd.arg(arg); + self + } + + /// Add any number of arguments to the command. + pub fn args(&mut self, args: I) -> &mut TestCommand + where + I: IntoIterator, + A: AsRef, + { + self.cmd.args(args); + self + } + + /// Set an environment variable for this command. + pub fn env( + &mut self, + key: impl AsRef, + val: impl AsRef, + ) -> &mut TestCommand { + self.cmd.env(key, val); + self + } + + /// Set the working directory for this command. + /// + /// The path given is interpreted relative to the directory that this + /// command was created for. + /// + /// Note that this does not need to be called normally, since the creation + /// of this TestCommand causes its working directory to be set to the + /// test's directory automatically. + pub fn current_dir>(&mut self, dir: P) -> &mut TestCommand { + self.cmd.current_dir(self.dir.path().join(dir)); + self + } + + /// Runs and captures the stdout of the given command. + pub fn stdout(&mut self) -> String { + let o = self.output(); + String::from_utf8_lossy(&o.stdout).into_owned() + } + + /// Pipe `input` to a command, and collect the output. + pub fn pipe(&mut self, input: &[u8]) -> String { + self.cmd.stdin(process::Stdio::piped()); + self.cmd.stdout(process::Stdio::piped()); + self.cmd.stderr(process::Stdio::piped()); + + let mut child = self.cmd.spawn().unwrap(); + + // Pipe input to child process using a separate thread to avoid + // risk of deadlock between parent and child process. + let mut stdin = child.stdin.take().expect("expected standard input"); + let input = input.to_owned(); + let worker = thread::spawn(move || stdin.write_all(&input)); + + let output = self.expect_success(child.wait_with_output().unwrap()); + worker.join().unwrap().unwrap(); + + String::from_utf8_lossy(&output.stdout).into_owned() + } + + /// Gets the output of a command. If the command failed, then this panics. + pub fn output(&mut self) -> process::Output { + let output = self.raw_output(); + self.expect_success(output) + } + + /// Gets the raw output of a command after filtering nonsense like jemalloc + /// error messages from stderr. + pub fn raw_output(&mut self) -> process::Output { + let mut output = self.cmd.output().unwrap(); + output.stderr = strip_jemalloc_nonsense(&output.stderr); + output + } + + /// Runs the command and asserts that it resulted in an error exit code. + pub fn assert_err(&mut self) { + let o = self.raw_output(); + if o.status.success() { + panic!( + "\n\n===== {:?} =====\n\ + command succeeded but expected failure!\ + \n\ncwd: {}\ + \n\ndir list: {:?}\ + \n\nstatus: {}\ + \n\nstdout: {}\n\nstderr: {}\ + \n\n=====\n", + self.cmd, + self.dir.dir.display(), + dir_list(&self.dir.dir), + o.status, + String::from_utf8_lossy(&o.stdout), + String::from_utf8_lossy(&o.stderr) + ); + } + } + + /// Runs the command and asserts that its exit code matches expected exit + /// code. + pub fn assert_exit_code(&mut self, expected_code: i32) { + let code = self.cmd.output().unwrap().status.code().unwrap(); + assert_eq!( + expected_code, + code, + "\n\n===== {:?} =====\n\ + expected exit code did not match\ + \n\ncwd: {}\ + \n\ndir list: {:?}\ + \n\nexpected: {}\ + \n\nfound: {}\ + \n\n=====\n", + self.cmd, + self.dir.dir.display(), + dir_list(&self.dir.dir), + expected_code, + code + ); + } + + /// Runs the command and asserts that something was printed to stderr. + pub fn assert_non_empty_stderr(&mut self) { + let o = self.cmd.output().unwrap(); + if o.status.success() || o.stderr.is_empty() { + panic!( + "\n\n===== {:?} =====\n\ + command succeeded but expected failure!\ + \n\ncwd: {}\ + \n\ndir list: {:?}\ + \n\nstatus: {}\ + \n\nstdout: {}\n\nstderr: {}\ + \n\n=====\n", + self.cmd, + self.dir.dir.display(), + dir_list(&self.dir.dir), + o.status, + String::from_utf8_lossy(&o.stdout), + String::from_utf8_lossy(&o.stderr) + ); + } + } + + fn expect_success(&self, o: process::Output) -> process::Output { + if !o.status.success() { + let suggest = if o.stderr.is_empty() { + "\n\nDid your search end up with no results?".to_string() + } else { + "".to_string() + }; + + panic!( + "\n\n==========\n\ + command failed but expected success!\ + {}\ + \n\ncommand: {:?}\ + \n\ncwd: {}\ + \n\ndir list: {:?}\ + \n\nstatus: {}\ + \n\nstdout: {}\ + \n\nstderr: {}\ + \n\n==========\n", + suggest, + self.cmd, + self.dir.dir.display(), + dir_list(&self.dir.dir), + o.status, + String::from_utf8_lossy(&o.stdout), + String::from_utf8_lossy(&o.stderr) + ); + } + o + } +} + +fn nice_err(path: &Path, res: Result) -> T { + match res { + Ok(t) => t, + Err(err) => panic!("{}: {:?}", path.display(), err), + } +} + +fn repeat io::Result<()>>(mut f: F) -> io::Result<()> { + let mut last_err = None; + for _ in 0..10 { + if let Err(err) = f() { + last_err = Some(err); + thread::sleep(Duration::from_millis(500)); + } else { + return Ok(()); + } + } + Err(last_err.unwrap()) +} + +/// Return a recursive listing of all files and directories in the given +/// directory. This is useful for debugging transient and odd failures in +/// integration tests. +fn dir_list>(dir: P) -> Vec { + walkdir::WalkDir::new(dir) + .follow_links(true) + .into_iter() + .map(|result| result.unwrap().path().to_string_lossy().into_owned()) + .collect() +} + +/// When running tests with cross, we need to be a bit smarter about how we +/// run our `rg` binary. We can't just run it directly since it might be +/// compiled for a totally different target. Instead, it's likely that `cross` +/// will have setup qemu to run it. While this is integrated into the Rust +/// testing by default, we need to handle it ourselves for integration tests. +/// +/// Now thankfully, cross sets `CROSS_RUNNER` to point to the right qemu +/// executable. Or so one thinks. But it seems to always be set to `qemu-user` +/// and I cannot find `qemu-user` anywhere in the Docker image. Awesome. +/// +/// Thers is `/linux-runner` which seems to work sometimes? But not always. +/// +/// Instead, it looks like we have to use `qemu-aarch64` in the `aarch64` +/// case. Perfect, so just get the current target architecture and append it +/// to `qemu-`. Wrong. Cross (or qemu or whoever) uses `qemu-ppc64` for +/// `powerpc64`, so we can't just use the target architecture as Rust knows +/// it verbatim. +/// +/// So... we just manually handle these cases. So fucking fun. +fn cross_runner() -> Option { + let runner = std::env::var("CROSS_RUNNER").ok()?; + if runner.is_empty() || runner == "empty" { + return None; + } + if cfg!(target_arch = "powerpc64") { + Some("qemu-ppc64".to_string()) + } else if cfg!(target_arch = "x86") { + Some("i386".to_string()) + } else { + // Make a guess... Sigh. + Some(format!("qemu-{}", std::env::consts::ARCH)) + } +} + +/// Returns true if the test setup believes Cross is running and `qemu` is +/// needed to run ripgrep. +/// +/// This is useful because it has been difficult to get some tests to pass +/// under Cross. +pub fn is_cross() -> bool { + std::env::var("CROSS_RUNNER").ok().map_or(false, |v| !v.is_empty()) +} + +/// Strips absolutely fucked `:` lines from the output. +/// +/// In theory this only happens under qemu, which is where our tests run under +/// `cross`. But is messes with our tests, because... they don't expect the +/// allocator to fucking write to stderr. I mean, what the fuck? Who prints a +/// warning message with absolutely no instruction for what to do with it or +/// how to disable it. Absolutely fucking bonkers. +fn strip_jemalloc_nonsense(data: &[u8]) -> Vec { + let lines = data + .lines_with_terminator() + .filter(|line| !line.starts_with_str(":")); + bstr::concat(lines) +} diff --git a/userland/search-test.nu b/userland/search-test.nu new file mode 100644 index 0000000..62548db --- /dev/null +++ b/userland/search-test.nu @@ -0,0 +1,102 @@ +# Compare stdout and numeric statuses on one fixture tree. Run after the release build. +def main [] { + $env.LC_ALL = 'C' + let repo = ($env.FILE_PWD | path dirname) + let artifacts = ($repo | path join userland target tools wasm32-wasip2 release) + let host = ($repo | path join target tools debug sausage) + let world = (mktemp -d) + let fixture = ($world | path join world) + mkdir ($fixture | path join sub) ($fixture | path join bin) + "alpha\nBeta alpha\nwords alphabet\nother\nalpha alpha\n" | save ($fixture | path join a.txt) + "other\nalpha\n" | save ($fixture | path join sub b.txt) + "different\n" | save ($fixture | path join c.log) + mkdir ($fixture | path join left nested) ($fixture | path join right nested) + "before\n" | save ($fixture | path join left nested a.txt) + "after\n" | save ($fixture | path join right nested a.txt) + "only left\n" | save ($fixture | path join left only.txt) + '{"items":[1,2,3]}' | save ($fixture | path join data.json) + for tool in [rg grep sed fd jaq find xargs env timeout] { + cp ($artifacts | path join $"($tool).wasm") ($fixture | path join bin $tool) + } + for tool in [diff cmp] { cp ($artifacts | path join diffutils.wasm) ($fixture | path join bin $tool) } + for tool in [echo wc printf printenv sleep] { + cp ($repo | path join userland target wasm32-wasip2 release sausage-coreutils.wasm) ($fixture | path join bin $tool) + } + cp ($repo | path join userland brush source target wasm32-wasip2 release brush.wasm) ($fixture | path join bin sh) + let cases = [ + {tool: grep, args: [-rn alpha sub]} + {tool: grep, args: [-i beta a.txt]} + {tool: grep, args: [-v alpha a.txt]} + {tool: grep, args: [-l alpha a.txt sub/b.txt]} + {tool: grep, args: [-c alpha a.txt]} + {tool: grep, args: [-w alpha a.txt]} + {tool: grep, args: [-E 'alpha|Beta' a.txt]} + {tool: grep, args: [-F 'alpha alpha' a.txt]} + {tool: grep, args: [-o alpha a.txt]} + {tool: grep, args: [-A 1 Beta a.txt]} + {tool: grep, args: [-B 1 Beta a.txt]} + {tool: grep, args: [-C 1 Beta a.txt]} + {tool: grep, args: [-r '--include=*.txt' alpha sub]} + {tool: grep, args: [absent a.txt]} + {tool: grep, args: [alpha missing]} + {tool: rg, args: [--sort path alpha a.txt]} + {tool: rg, args: [--sort path -g '*.txt' alpha sub]} + {tool: rg, args: [--sort path -l alpha sub]} + {tool: find, args: [sub -name '*.txt']} + {tool: find, args: [sub -type f]} + {tool: find, args: [sub -path 'sub/*.txt']} + {tool: find, args: [sub -maxdepth 0]} + {tool: find, args: [sub -newer a.txt]} + {tool: find, args: [sub -print0]} + {tool: sed, args: [-n '2p' a.txt]} + {tool: sed, args: ['s/alpha/ALPHA/g' a.txt]} + {tool: sed, args: [-E 's/(alpha)/[\1]/g' a.txt]} + {tool: sed, args: ['2,3d' a.txt]} + {tool: diff, args: [-u a.txt a.txt]} + {tool: diff, args: [-u a.txt c.log]} + {tool: diff, args: [-r sub sub]} + {tool: diff, args: [-r left right]} + {tool: cmp, args: [a.txt a.txt]} + {tool: cmp, args: [a.txt c.log]} + {tool: fd, args: [b sub]} + {tool: fd, args: [-e txt . sub]} + {tool: jaq, args: ['.items | map(. * 2)' data.json]} + ] + for case in $cases { + let native = if $case.tool == jaq { + do { cd $fixture; ^nix shell nixpkgs#jq -c jq ...$case.args } | complete + } else { do { cd $fixture; ^$case.tool ...$case.args } | complete } + let guest = if $case.tool == find { + let quoted = ($case.args | each {|a| $"'($a)'" } | str join ' ') + do { ^$host sh $world -c $"find ($quoted)" } | complete + } else { + let name = if $case.tool in [diff cmp] { 'diffutils' } else { $case.tool } + let wasm = ($artifacts | path join $"($name).wasm") + let args = if $name == diffutils { [$case.tool] | append $case.args } else { $case.args } + do { ^wasmtime run --dir $"($fixture)::/" --env PWD=/ $wasm ...$args } | complete + } + if $native.stdout != $guest.stdout or $native.exit_code != $guest.exit_code { + error make {msg: $"Mismatch ($case): native=($native) guest=($guest). Fixture: ($fixture)"} + } + print $"PASS ($case.tool) ($case.args)" + } + # In-place editing changes the fixture. Reset it before the guest comparison. + "alpha alpha\n" | save ($fixture | path join edit.txt) + let native = (do { cd $fixture; ^sed -i 's/alpha/B/g' edit.txt } | complete) + let expected = (open --raw ($fixture | path join edit.txt)) + "alpha alpha\n" | save -f ($fixture | path join edit.txt) + let guest = (do { ^wasmtime run --dir $"($fixture)::/" --env PWD=/ ($artifacts | path join sed.wasm) -i 's/alpha/B/g' edit.txt } | complete) + if $guest.exit_code != $native.exit_code or (open --raw ($fixture | path join edit.txt)) != $expected { error make {msg: 'sed -i mismatch'} } + print 'PASS sed -i' + for case in [ + {script: 'env FOO=1 printenv FOO', stdout: "1\n", code: 0} + {script: 'timeout 1 sleep 5', stdout: '', code: 124} + {script: 'cd sub && find . -name "*.txt" -exec wc -l {} \;', stdout: "2 ./b.txt\n", code: 0} + {script: "printf 'a\nb\n' | xargs -n1 echo", stdout: "a\nb\n", code: 0} + ] { + let result = (do { ^$host sh $world -c $case.script } | complete) + if $result.stdout != $case.stdout or $result.exit_code != $case.code { error make {msg: $"Spawn mismatch: ($case) got ($result)"} } + print $"PASS ($case.script)" + } + rm -r $world +} diff --git a/userland/sed/Cargo.toml b/userland/sed/Cargo.toml new file mode 100644 index 0000000..467bb48 --- /dev/null +++ b/userland/sed/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "sausage-sed" +version = "0.1.0" +edition = "2024" + +[[bin]] +name = "sed" +path = "src/main.rs" + +[dependencies] +sed = "=0.3.0" diff --git a/userland/sed/src/main.rs b/userland/sed/src/main.rs new file mode 100644 index 0000000..d52e804 --- /dev/null +++ b/userland/sed/src/main.rs @@ -0,0 +1,4 @@ +//! Run the pinned uutils sed library with the shared WASI startup object. +fn main() { + std::process::exit(sed::sed::uumain(std::env::args_os())); +} diff --git a/userland/spawn/Cargo.toml b/userland/spawn/Cargo.toml new file mode 100644 index 0000000..794a6a2 --- /dev/null +++ b/userland/spawn/Cargo.toml @@ -0,0 +1,7 @@ +[package] +name = "sausage-spawn" +version = "0.1.0" +edition = "2024" + +[dependencies] +wit-bindgen = "=0.46.0" diff --git a/userland/spawn/README.md b/userland/spawn/README.md new file mode 100644 index 0000000..4bc2849 --- /dev/null +++ b/userland/spawn/README.md @@ -0,0 +1,26 @@ +# World process library + +`sausage-spawn` starts WASI components through `sausage:process/host@0.1.0`. +It uses `wit-bindgen` 0.46.0 and the canonical `host/wit/process.wit`. +It adds no host interface. + +```rust,no_run +use sausage_spawn::Command; +let status = Command::new("printenv").arg("FOO").env("FOO", "1").status()?; +assert_eq!(status, 0); +# Ok::<(), std::io::Error>(()) +``` + +A command inherits the environment, cwd, and all three streams. +`Stdio::Pipe` exposes the generated WASI stream resource on `Child`. +`Stdio::Null` discards output or supplies EOF. +Drop piped stdin to supply EOF. +Drain both output pipes before `wait` to prevent a bounded-pipe deadlock. +`try_wait` returns the numeric status when the child finishes. +`kill` stops every process in the child's group through the process interface. +Use `Command::new_group` before spawn when the caller must survive a child kill. +`timeout` uses a separate child group. +Drop does not kill the child. + +Run `nix develop -c nu userland/spawn/check-imports.nu` to inspect the release components. +The script keeps the complete WIT output and checks each import. diff --git a/userland/spawn/check-imports.nu b/userland/spawn/check-imports.nu new file mode 100644 index 0000000..8e60c1b --- /dev/null +++ b/userland/spawn/check-imports.nu @@ -0,0 +1,18 @@ +# Keep the full WIT output as evidence, then check every declared import. +def main [] { + let repo = ($env.FILE_PWD | path dirname | path dirname) + let artifacts = ($repo | path join userland target tools wasm32-wasip2 release) + let evidence = (^mktemp -d --tmpdir=/tmp t14-imports.XXXXXXXX) + for tool in [rg grep find xargs sed diffutils fd jaq env timeout] { + let wasm = ($artifacts | path join $"($tool).wasm") + let wit = (^wasm-tools component wit $wasm) + $wit | save ($evidence | path join $"($tool).wit") + let imports = ($wit | lines | where {|line| $line | str starts-with ' import ' }) + for import in $imports { + let allowed = ($import | str starts-with ' import wasi:') or (($tool in [find xargs env timeout]) and ($import | str starts-with ' import sausage:process/host@0.1.0;')) + if not $allowed { error make {msg: $"Unexpected import: ($tool): ($import)"} } + } + print $"PASS imports ($tool): ((ls $wasm | get size | first | into int)) bytes" + } + print $"Full WIT evidence: ($evidence)" +} diff --git a/userland/spawn/src/lib.rs b/userland/spawn/src/lib.rs new file mode 100644 index 0000000..004ee87 --- /dev/null +++ b/userland/spawn/src/lib.rs @@ -0,0 +1,70 @@ +//! Launch world programs through the process interface. No native subprocesses. +//! Commands inherit the environment, cwd, and streams unless the caller changes them. +//! Pipe callers must drain stdout and stderr before wait to avoid bounded-pipe deadlocks. +use std::{collections::BTreeMap, io, path::{Path, PathBuf}}; +mod bindings { + wit_bindgen::generate!({ path: "../../host/wit", world: "program", generate_all }); +} +use bindings::sausage::process::host; +pub use bindings::wasi::io::streams::{InputStream, OutputStream}; +pub use host::Stdio; + +/// A prepared child process. +pub struct Command { + program: String, + args: Vec, + env: BTreeMap, + cwd: PathBuf, + group: Option, + stdin: Stdio, stdout: Stdio, stderr: Stdio, +} +impl Command { + /// Use the parent's cwd, environment, and streams. + pub fn new(program: impl AsRef) -> Self { + Self { program: program.as_ref().into(), args: Vec::new(), env: std::env::vars().collect(), + cwd: std::env::current_dir().unwrap_or_else(|_| "/".into()), group: None, + stdin: Stdio::Inherit, stdout: Stdio::Inherit, stderr: Stdio::Inherit } + } + pub fn arg(&mut self, arg: impl AsRef) -> &mut Self { self.args.push(arg.as_ref().into()); self } + pub fn args(&mut self, args: impl IntoIterator>) -> &mut Self { + for arg in args { self.arg(arg); } self + } + pub fn env(&mut self, key: impl AsRef, value: impl AsRef) -> &mut Self { + self.env.insert(key.as_ref().into(), value.as_ref().into()); self + } + pub fn env_clear(&mut self) -> &mut Self { self.env.clear(); self } + pub fn current_dir(&mut self, cwd: impl AsRef) -> &mut Self { self.cwd = cwd.as_ref().into(); self } + /// Isolate the child group so kill does not stop the caller's group. + pub fn new_group(&mut self) -> &mut Self { self.group = Some(0); self } + pub fn stdin(&mut self, mode: Stdio) -> &mut Self { self.stdin = mode; self } + pub fn stdout(&mut self, mode: Stdio) -> &mut Self { self.stdout = mode; self } + pub fn stderr(&mut self, mode: Stdio) -> &mut Self { self.stderr = mode; self } + /// Resolve PATH and start the component. + pub fn spawn(&self) -> io::Result { + let program = if self.program.contains('/') { self.program.clone() } else { + self.env.get("PATH").map(String::as_str).unwrap_or("/usr/local/bin:/bin").split(':') + .map(|dir| self.cwd.join(dir).join(&self.program)) + .find(|p| p.is_file()).ok_or_else(|| io::Error::new(io::ErrorKind::NotFound, &self.program[..]))? + .to_string_lossy().into_owned() + }; + let child = host::spawn(&host::Request { program, argv: std::iter::once(self.program.clone()).chain(self.args.clone()).collect(), + env: self.env.clone().into_iter().collect(), cwd: self.cwd.to_string_lossy().into_owned(), group: self.group, + stdin: self.stdin, stdout: self.stdout, stderr: self.stderr }).map_err(io::Error::other)?; + Ok(Child { stdin: matches!(self.stdin, Stdio::Pipe).then(|| child.stdin()), + stdout: matches!(self.stdout, Stdio::Pipe).then(|| child.stdout()), + stderr: matches!(self.stderr, Stdio::Pipe).then(|| child.stderr()), child }) + } + pub fn status(&self) -> io::Result { Ok(self.spawn()?.wait()) } +} +/// A child handle. Drop does not kill the child. Drop stdin to send EOF. +pub struct Child { + child: host::Child, + pub stdin: Option, pub stdout: Option, pub stderr: Option, +} +impl Child { + pub fn id(&self) -> u32 { self.child.pid() } + pub fn try_wait(&self) -> Option { self.child.status() } + pub fn wait(&self) -> i32 { self.child.wait() } + /// Stop every process in this child's group. + pub fn kill(&self) { self.child.kill(); } +}