haeaeae/proxy-scraper-checker
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor: streamline allocator benchmark scripts for improved readability and performance

Browse Source
This commit is contained in:
monosans
2026-04-30 16:46:21 +03:00
parent 5f1cdd3e48
commit ebc6c98a86
3 changed files with 39 additions and 89 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/scripts/allocator_bench_alpine.sh
+15 -27
View File
@@ -6,7 +6,7 @@ if [[ "${TOKIO_MULTI_THREAD:-false}" == "true" ]]; then
TOKIO_FEATURE="tokio-multi-thread"
fi
ALPINE_SCRIPT=$(cat <<'EOF'
ALPINE_SCRIPT=$(cat <<'INNER_EOF'
set -eu
apk add --no-cache build-base pkgconfig time rust cargo
@@ -34,27 +34,23 @@ for allocator in system jemalloc mimalloc_v2 mimalloc_v3; do
else
cargo build --release --locked
fi
if [ -n "$features" ]; then
output="$(/usr/bin/time -v /work/target/release/proxy-scraper-checker 2>&1 >/dev/null)"
else
output="$(/usr/bin/time -v /work/target/release/proxy-scraper-checker 2>&1 >/dev/null)"
fi
peak="$(echo "$output" | awk -F': ' '/Maximum resident set size/ {print $2; exit}')"
major="$(echo "$output" | awk -F': ' '/Major \(requiring I\/O\) page faults/ {print $2; exit}')"
minor="$(echo "$output" | awk -F': ' '/Minor \(reclaiming a frame\) page faults/ {print $2; exit}')"
output="$(/usr/bin/time -v /work/target/release/proxy-scraper-checker 2>&1 >/dev/null)"
peak="$(echo "$output" | awk -F': ' '/Maximum resident set size/ {print $2; exit}')"
major="$(echo "$output" | awk -F': ' '/Major \(requiring I\/O\) page faults/ {print $2; exit}')"
minor="$(echo "$output" | awk -F': ' '/Minor \(reclaiming a frame\) page faults/ {print $2; exit}')"
if [ -z "$peak" ]; then
echo "Failed to parse peak memory for $allocator" >&2
exit 1
fi
if [ -z "$major" ]; then
major=0
fi
if [ -z "$minor" ]; then
minor=0
fi
printf "%s\t%s\t%s\t%s\n" "$allocator" "$peak" "$major" "$minor" >> /work/alpine-results.tsv
major="${major:-0}"
minor="${minor:-0}"
printf "%s\t%s\t%s\t%s\n" "$allocator" "$peak" "$major" "$minor" >> /work/alpine-results.tsv
done
EOF
INNER_EOF
)
docker run --rm \
@@ -69,16 +65,8 @@ docker run --rm \
echo ""
echo "| Allocator | Peak KB | Major PF | Minor PF |"
echo "| --- | ---: | ---: | ---: |"
sort -n -k2,2 -k3,3 -k4,4 alpine-results.tsv | while IFS=$'\t' read -r allocator peak major minor; do
while IFS=$'\t' read -r allocator peak major minor; do
echo "| $allocator | $peak | $major | $minor |"
done
best="$(sort -n -k2,2 -k3,3 -k4,4 alpine-results.tsv | head -n1)"
best_allocator="${best%%$'\t'*}"
best_rest="${best#*$'\t'}"
best_peak="${best_rest%%$'\t'*}"
best_rest="${best_rest#*$'\t'}"
best_major="${best_rest%%$'\t'*}"
best_minor="${best_rest#*$'\t'}"
done < alpine-results.tsv
echo ""
echo "**Best:** $best_allocator ($best_peak KB, $best_major major PF, $best_minor minor PF)"
} >> "$GITHUB_STEP_SUMMARY"
.github/scripts/allocator_bench_unix.sh
+11 -34
View File
@@ -10,26 +10,14 @@ allocators=(system jemalloc mimalloc_v2 mimalloc_v3)
if [[ "$RUNNER_OS" == "Linux" ]]; then
time_cmd=(/usr/bin/time -v)
parse_peak() {
awk -F': ' '/Maximum resident set size/ {print $2; exit}'
}
parse_major() {
awk -F': ' '/Major \(requiring I\/O\) page faults/ {print $2; exit}'
}
parse_minor() {
awk -F': ' '/Minor \(reclaiming a frame\) page faults/ {print $2; exit}'
}
parse_peak() { awk -F': ' '/Maximum resident set size/ {print $2; exit}'; }
parse_major() { awk -F': ' '/Major \(requiring I\/O\) page faults/ {print $2; exit}'; }
parse_minor() { awk -F': ' '/Minor \(reclaiming a frame\) page faults/ {print $2; exit}'; }
else
time_cmd=(/usr/bin/time -l)
parse_peak() {
awk '/maximum resident set size/ {print $1; exit}'
}
parse_major() {
awk '/page faults/ {print $1; exit}'
}
parse_minor() {
awk '/page reclaims/ {print $1; exit}'
}
parse_peak() { awk '/maximum resident set size/ {print $1; exit}'; }
parse_major() { awk '/page faults/ {print $1; exit}'; }
parse_minor() { awk '/page reclaims/ {print $1; exit}'; }
fi
build_features() {
@@ -74,16 +62,13 @@ run_one() {
major="$(echo "$output" | parse_major)"
local minor
minor="$(echo "$output" | parse_minor)"
if [[ -z "$peak" ]]; then
echo "Failed to parse peak memory for $allocator" >&2
exit 1
fi
if [[ -z "$major" ]]; then
major=0
fi
if [[ -z "$minor" ]]; then
minor=0
fi
major="${major:-0}"
minor="${minor:-0}"
if [[ "$RUNNER_OS" != "Linux" ]]; then
peak=$((peak / 1024))
@@ -106,16 +91,8 @@ done
echo ""
echo "| Allocator | Peak KB | Major PF | Minor PF |"
echo "| --- | ---: | ---: | ---: |"
sort -n -k2,2 -k3,3 -k4,4 results.tsv | while IFS=$'\t' read -r allocator peak major minor; do
while IFS=$'\t' read -r allocator peak major minor; do
echo "| $allocator | $peak | $major | $minor |"
done
best="$(sort -n -k2,2 -k3,3 -k4,4 results.tsv | head -n1)"
best_allocator="${best%%$'\t'*}"
best_rest="${best#*$'\t'}"
best_peak="${best_rest%%$'\t'*}"
best_rest="${best_rest#*$'\t'}"
best_major="${best_rest%%$'\t'*}"
best_minor="${best_rest#*$'\t'}"
done < results.tsv
echo ""
echo "**Best:** $best_allocator ($best_peak KB, $best_major major PF, $best_minor minor PF)"
} >> "$GITHUB_STEP_SUMMARY"
.github/scripts/allocator_bench_windows.ps1
+13 -28
View File
@@ -13,24 +13,18 @@ function Build-Features([string]$allocator, [string]$tokio) {
function Run-One([string]$allocator, [string]$features) {
$args = @("build", "--release", "--locked")
if ($features) { $args += "--features"; $args += $features }
$proc = Start-Process -FilePath "cargo" -ArgumentList $args -PassThru -NoNewWindow -Wait
$peak = 0
$proc = Start-Process -FilePath "cargo" -ArgumentList $args -PassThru -NoNewWindow
while (-not $proc.HasExited) {
Start-Sleep -Milliseconds 200
try {
$p = Get-Process -Id $proc.Id -ErrorAction Stop
$current = [math]::Max($p.WorkingSet64, $p.PeakWorkingSet64)
if ($current -gt $peak) { $peak = $current }
} catch { }
}
$exe = "target\\release\\proxy-scraper-checker.exe"
$exe = "target\release\proxy-scraper-checker.exe"
$proc = Start-Process -FilePath $exe -PassThru -NoNewWindow
$peak = 0
$faults = 0
$proc = Start-Process -FilePath $exe -PassThru -NoNewWindow
while (-not $proc.HasExited) {
Start-Sleep -Milliseconds 200
Start-Sleep -Milliseconds 100
try {
$p = Get-Process -Id $proc.Id -ErrorAction Stop
$current = [math]::Max($p.WorkingSet64, $p.PeakWorkingSet64)
@@ -38,6 +32,7 @@ function Run-One([string]$allocator, [string]$features) {
if ($p.PageFaults -gt $faults) { $faults = $p.PageFaults }
} catch { }
}
$peakKb = [math]::Floor($peak / 1kb)
Add-Content -Path results.tsv -Value "$allocator`t$peakKb`t$faults"
}
@@ -48,25 +43,15 @@ foreach ($allocator in $allocators) {
Run-One $allocator $features
}
$rows = Get-Content results.tsv | ForEach-Object {
$parts = $_ -split "`t"
[pscustomobject]@{
Allocator = $parts[0]
PeakKB = [int]$parts[1]
PageFaults = [int]$parts[2]
}
}
$sorted = $rows | Sort-Object PeakKB, PageFaults
$best = $sorted | Select-Object -First 1
$summary = @()
$summary += "### $($env:PLATFORM_LABEL) (tokio-multi-thread=$tokioOn)"
$summary += ""
$summary += "| Allocator | Peak KB | Page Faults |"
$summary += "| --- | ---: | ---: |"
foreach ($row in $sorted) {
$summary += "| $($row.Allocator) | $($row.PeakKB) | $($row.PageFaults) |"
Get-Content results.tsv | ForEach-Object {
$parts = $_ -split "`t"
$summary += "| $($parts[0]) | $($parts[1]) | $($parts[2]) |"
}
$summary += ""
$summary += "**Best:** $($best.Allocator) ($($best.PeakKB) KB, $($best.PageFaults) page faults)"
$summary -join "`n" | Add-Content $env:GITHUB_STEP_SUMMARY