surma/sausage
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

userland: add archive tools and pinned GoAWK, esbuild, and QuickJS

Browse Source
This commit is contained in:
surma
2026-11-07 10:56:29 +00:00
parent 7e7e49bfa2
commit d79b2bc2e1
1798 changed files with 562969 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
userland/goawk/source/scripts/benchmark.sh
Executable
+2
View File
@@ -0,0 +1,2 @@
#!/bin/sh
go test ./interp -bench=. -count=5 > benchmarks_new.txt
userland/goawk/source/scripts/benchmark_awks.py
Executable
+124
View File
@@ -0,0 +1,124 @@
#!/usr/bin/env python3
# Benchmark GoAWK against other AWK versions
from __future__ import print_function
import glob
import os.path
import shutil
import subprocess
import sys
import time
AWKS = [
'./goawk',
'./orig', # GoAWK without perf improvements
'original-awk',
'gawk',
'mawk',
]
NORM_INDEX = AWKS.index('original-awk')
TESTS_TO_MEAN = None # By default, calculate the mean of all tests
if False:
# Only get the mean of these tests because these are the only ones
# we show in the GoAWK article.
TESTS_TO_MEAN = [
'tt.01_print',
'tt.02_print_NR_NF',
'tt.02a_print_length',
'tt.03_sum_length',
'tt.03a_sum_field',
'tt.04_printf_fields',
'tt.05_concat_fields',
'tt.06_count_lengths',
'tt.07_even_fields',
'tt.big_complex_program',
'tt.x1_mandelbrot',
'tt.x2_sum_loop',
]
NUM_RUNS = 6
MIN_TIME = 0.5
PROGRAM_GLOB = 'testdata/tt.*'
if len(sys.argv) > 1:
PROGRAM_GLOB = 'testdata/' + sys.argv[1]
def repeat_file(input_file, repeated_file, n):
with open(input_file, 'rb') as fin, open(repeated_file, 'wb') as fout:
for i in range(n):
fin.seek(0)
shutil.copyfileobj(fin, fout)
print('Test ', end='')
for awk in AWKS:
display_awk = os.path.basename(awk)
display_awk = display_awk.replace('original-awk', 'awk')
print('| {:>8} '.format(display_awk), end='')
print()
print('-'*28 + ' | --------'*len(AWKS))
repeats_created = []
products = [1] * len(AWKS)
num_products = 0
programs = sorted(glob.glob(PROGRAM_GLOB))
for program in programs:
# First do a test run with GoAWK to see roughly how long it takes
cmdline = '{} -f {} testdata/foo.td >tt.out'.format(AWKS[0], program)
start = time.time()
status = subprocess.call(cmdline, shell=True)
elapsed = time.time() - start
# If test run took less than MIN_TIME seconds, scale/repeat input
# file accordingly
input_file = 'testdata/foo.td'
if elapsed < MIN_TIME:
multiplier = int(round(MIN_TIME / elapsed))
repeated_file = '{}.{}'.format(input_file, multiplier)
if not os.path.exists(repeated_file):
repeat_file(input_file, repeated_file, multiplier)
repeats_created.append(repeated_file)
input_file = repeated_file
# Record time taken to run this test, running each NUM_RUMS times
# and taking the minimum elapsed time
awk_times = []
for awk in AWKS:
cmdline = '{} -f {} {} >tt.out'.format(awk, program, input_file)
times = []
for i in range(NUM_RUNS):
start = time.time()
status = subprocess.call(cmdline, shell=True)
elapsed = time.time() - start
times.append(elapsed)
if status != 0:
print('ERROR status {} from cmd: {}'.format(status, cmdline), file=sys.stderr)
min_time = min(sorted(times)[1:])
awk_times.append(min_time)
# Normalize to One True AWK time = 1.0
norm_time = awk_times[NORM_INDEX]
speeds = [norm_time/t for t in awk_times]
test_name = program.split('/')[1]
if TESTS_TO_MEAN is None or test_name in TESTS_TO_MEAN:
num_products += 1
for i in range(len(AWKS)):
products[i] *= speeds[i]
display_name = test_name.split('_')[0] + ' (' + ' '.join(test_name.split('_')[1:]) + ')'
print('{:28}'.format(display_name), end='')
for i, awk in enumerate(AWKS):
print(' | {:8.2f}'.format(speeds[i]), end='')
print()
print('-'*28 + ' | --------'*len(AWKS))
print('**Geo mean** ', end='')
for i, awk in enumerate(AWKS):
print(' | **{:.2f}**'.format(products[i] ** (1.0/num_products)), end='')
print()
# Delete temporary files created
os.remove('tt.out')
for repeated_file in repeats_created:
os.remove(repeated_file)
userland/goawk/source/scripts/benchstat.sh
Executable
+2
View File
@@ -0,0 +1,2 @@
#!/bin/sh
~/go/bin/benchstat -sort=delta -geomean benchmarks_old.txt benchmarks_new.txt
userland/goawk/source/scripts/csvbench/count.py
+9
View File
@@ -0,0 +1,9 @@
import csv
import sys
lines, fields = 0, 0
for row in csv.reader(sys.stdin):
lines += 1
fields += len(row)
print(lines, fields)
userland/goawk/source/scripts/csvbench/count/main.go
+27
View File
@@ -0,0 +1,27 @@
package main
import (
"bufio"
"encoding/csv"
"fmt"
"io"
"log"
"os"
)
func main() {
reader := csv.NewReader(bufio.NewReader(os.Stdin))
lines, fields := 0, 0
for {
row, err := reader.Read()
if err == io.EOF {
break
}
if err != nil {
log.Fatal(err)
}
lines++
fields += len(row)
}
fmt.Println(lines, fields)
}
userland/goawk/source/scripts/csvbench/csvbench.sh
Executable
+48
View File
@@ -0,0 +1,48 @@
#!/bin/sh
set -e
echo ===== Writing 1GB - goawk
time goawk -o csv 'BEGIN { for (i=0; i<3514073; i++) print i, "foo", "bob@example.com", "simple,quoted", "quoted string with \" in it", "0123456789", "9876543210", "The quick brown fox jumps over the lazy dog", "", "final field", i, "foo", "bob@example.com", "simple,quoted", "quoted string with \" in it", "0123456789", "9876543210", "The quick brown fox jumps over the lazy dog", "", "final field" }' >/dev/null
time goawk -o csv 'BEGIN { for (i=0; i<3514073; i++) print i, "foo", "bob@example.com", "simple,quoted", "quoted string with \" in it", "0123456789", "9876543210", "The quick brown fox jumps over the lazy dog", "", "final field", i, "foo", "bob@example.com", "simple,quoted", "quoted string with \" in it", "0123456789", "9876543210", "The quick brown fox jumps over the lazy dog", "", "final field" }' >/dev/null
time goawk -o csv 'BEGIN { for (i=0; i<3514073; i++) print i, "foo", "bob@example.com", "simple,quoted", "quoted string with \" in it", "0123456789", "9876543210", "The quick brown fox jumps over the lazy dog", "", "final field", i, "foo", "bob@example.com", "simple,quoted", "quoted string with \" in it", "0123456789", "9876543210", "The quick brown fox jumps over the lazy dog", "", "final field" }' >/dev/null
echo ===== Writing 1GB - frawk
time frawk -o csv 'BEGIN { for (i=0; i<3514073; i++) print i, "foo", "bob@example.com", "simple,quoted", "quoted string with \" in it", "0123456789", "9876543210", "The quick brown fox jumps over the lazy dog", "", "final field", i, "foo", "bob@example.com", "simple,quoted", "quoted string with \" in it", "0123456789", "9876543210", "The quick brown fox jumps over the lazy dog", "", "final field"; }' >/dev/null
time frawk -o csv 'BEGIN { for (i=0; i<3514073; i++) print i, "foo", "bob@example.com", "simple,quoted", "quoted string with \" in it", "0123456789", "9876543210", "The quick brown fox jumps over the lazy dog", "", "final field", i, "foo", "bob@example.com", "simple,quoted", "quoted string with \" in it", "0123456789", "9876543210", "The quick brown fox jumps over the lazy dog", "", "final field"; }' >/dev/null
time frawk -o csv 'BEGIN { for (i=0; i<3514073; i++) print i, "foo", "bob@example.com", "simple,quoted", "quoted string with \" in it", "0123456789", "9876543210", "The quick brown fox jumps over the lazy dog", "", "final field", i, "foo", "bob@example.com", "simple,quoted", "quoted string with \" in it", "0123456789", "9876543210", "The quick brown fox jumps over the lazy dog", "", "final field"; }' >/dev/null
echo ===== Writing 1GB - Python
time python3 write.py >/dev/null
time python3 write.py >/dev/null
time python3 write.py >/dev/null
echo ===== Writing 1GB - Go
go build -o bin/write ./write
time ./bin/write >/dev/null
time ./bin/write >/dev/null
time ./bin/write >/dev/null
./bin/write >count.csv
echo ===== Reading 1GB - goawk
time goawk -i csv '{ w+=NF } END { print NR, w }' <count.csv
time goawk -i csv '{ w+=NF } END { print NR, w }' <count.csv
time goawk -i csv '{ w+=NF } END { print NR, w }' <count.csv
echo ===== Reading 1GB - frawk
time frawk -i csv '{ w+=NF } END { print NR, w }' <count.csv
time frawk -i csv '{ w+=NF } END { print NR, w }' <count.csv
time frawk -i csv '{ w+=NF } END { print NR, w }' <count.csv
echo ===== Reading 1GB - Python
time python3 count.py <count.csv
time python3 count.py <count.csv
time python3 count.py <count.csv
echo ===== Reading 1GB - Go
go build -o bin/count ./count
time ./bin/count <count.csv
time ./bin/count <count.csv
time ./bin/count <count.csv
userland/goawk/source/scripts/csvbench/write.py
+27
View File
@@ -0,0 +1,27 @@
import csv
import sys
writer = csv.writer(sys.stdout)
for i in range(3514073): # will create a ~1GB file
writer.writerow([
i,
"foo",
"bob@example.com",
"simple,quoted",
"quoted string with \" in it",
"0123456789",
"9876543210",
"The quick brown fox jumps over the lazy dog",
"",
"final field",
i,
"foo",
"bob@example.com",
"simple,quoted",
"quoted string with \" in it",
"0123456789",
"9876543210",
"The quick brown fox jumps over the lazy dog",
"",
"final field",
])
userland/goawk/source/scripts/csvbench/write/main.go
+43
View File
@@ -0,0 +1,43 @@
package main
import (
"encoding/csv"
"log"
"os"
"strconv"
)
func main() {
writer := csv.NewWriter(os.Stdout)
for i := 0; i < 3514073; i++ { // will create a ~1GB file
err := writer.Write([]string{
strconv.Itoa(i),
"foo",
"bob@example.com",
"simple,quoted",
"quoted string with \" in it",
"0123456789",
"9876543210",
"The quick brown fox jumps over the lazy dog",
"",
"final field",
strconv.Itoa(i),
"foo",
"bob@example.com",
"simple,quoted",
"quoted string with \" in it",
"0123456789",
"9876543210",
"The quick brown fox jumps over the lazy dog",
"",
"final field",
})
if err != nil {
log.Fatal(err)
}
}
writer.Flush()
if writer.Error() != nil {
log.Fatal(writer.Error())
}
}
userland/goawk/source/scripts/fuzz_input.sh
Executable
+2
View File
@@ -0,0 +1,2 @@
#!/bin/sh
go test ./interp -run=^$ -fuzz=Input -parallel=4
userland/goawk/source/scripts/fuzz_source.sh
Executable
+2
View File
@@ -0,0 +1,2 @@
#!/bin/sh
go test ./interp -run=^$ -fuzz=Source -parallel=4
userland/goawk/source/scripts/make_binaries.sh
Executable
+31
View File
@@ -0,0 +1,31 @@
#!/bin/sh
# Create PGO profile
go test -run=^$ -bench=. -cpuprofile=default.pgo ./interp
go build
VERSION="$(./goawk -version)"
GOOS=windows GOARCH=386 go build -ldflags="-w"
zip "goawk_${VERSION}_windows_386.zip" goawk.exe README.md LICENSE.txt docs/*
GOOS=windows GOARCH=amd64 go build -ldflags="-w"
zip "goawk_${VERSION}_windows_amd64.zip" goawk.exe README.md LICENSE.txt docs/*
GOOS=linux GOARCH=386 go build -ldflags="-w"
tar -cvzf "goawk_${VERSION}_linux_386.tar.gz" goawk README.md LICENSE.txt docs/*
GOOS=linux GOARCH=amd64 go build -ldflags="-w"
tar -cvzf "goawk_${VERSION}_linux_amd64.tar.gz" goawk README.md LICENSE.txt docs/*
GOOS=linux GOARCH=arm64 go build -ldflags="-w"
tar -cvzf "goawk_${VERSION}_linux_arm64.tar.gz" goawk README.md LICENSE.txt docs/*
GOOS=darwin GOARCH=amd64 go build -ldflags="-w"
tar -cvzf "goawk_${VERSION}_darwin_amd64.tar.gz" goawk README.md LICENSE.txt docs/*
GOOS=darwin GOARCH=arm64 go build -ldflags="-w"
tar -cvzf "goawk_${VERSION}_darwin_arm64.tar.gz" goawk README.md LICENSE.txt docs/*
GOOS=freebsd GOARCH=amd64 go build -ldflags="-w"
tar -cvzf "goawk_${VERSION}_freebsd_amd64.tar.gz" goawk README.md LICENSE.txt docs/*
GOOS=freebsd GOARCH=arm go build -ldflags="-w"
tar -cvzf "goawk_${VERSION}_freebsd_arm.tar.gz" goawk README.md LICENSE.txt docs/*
rm -f goawk goawk.exe