haeaeae/proxy-scraper-checker
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: enhance allocator support across scripts and workflows

Browse Source
This commit is contained in:
monosans
2026-04-30 20:57:47 +03:00
parent c4a793ce69
commit 0e18571448
4 changed files with 114 additions and 141 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/scripts/allocator_bench_alpine.sh
+28 -34
View File
@@ -10,46 +10,39 @@ ALPINE_SCRIPT=$(cat <<'INNER_EOF'
set -eu
apk add --no-cache build-base pkgconfig time rust cargo
build_features() {
allocator="$1"
features=""
if [ "$allocator" != "system" ]; then
features="$allocator"
features=""
if [ "$ALLOCATOR" != "system" ]; then
features="$ALLOCATOR"
fi
if [ -n "$TOKIO_FEATURE" ]; then
if [ -n "$features" ]; then
features="$features,$TOKIO_FEATURE"
else
features="$TOKIO_FEATURE"
fi
if [ -n "$TOKIO_FEATURE" ]; then
if [ -n "$features" ]; then
features="$features,$TOKIO_FEATURE"
else
features="$TOKIO_FEATURE"
fi
fi
echo "$features"
}
fi
: > /work/alpine-results.tsv
for allocator in system jemalloc mimalloc_v2 mimalloc_v3; do
features="$(build_features "$allocator")"
if [ -n "$features" ]; then
cargo build --release --locked --features "$features"
else
cargo build --release --locked
fi
if [ -n "$features" ]; then
cargo build --release --locked --features "$features"
else
cargo build --release --locked
fi
output="$(/usr/bin/time -v /work/target/release/proxy-scraper-checker 2>&1 >/dev/null)"
output="$(/usr/bin/time -v /work/target/release/proxy-scraper-checker 2>&1 >/dev/null)"
peak="$(echo "$output" | awk -F': ' '/Maximum resident set size/ {print $2; exit}')"
major="$(echo "$output" | awk -F': ' '/Major \(requiring I\/O\) page faults/ {print $2; exit}')"
minor="$(echo "$output" | awk -F': ' '/Minor \(reclaiming a frame\) page faults/ {print $2; exit}')"
peak="$(echo "$output" | awk -F': ' '/Maximum resident set size/ {print $2; exit}')"
major="$(echo "$output" | awk -F': ' '/Major \(requiring I\/O\) page faults/ {print $2; exit}')"
minor="$(echo "$output" | awk -F': ' '/Minor \(reclaiming a frame\) page faults/ {print $2; exit}')"
if [ -z "$peak" ]; then
echo "Failed to parse peak memory for $allocator" >&2
exit 1
fi
major="${major:-0}"
minor="${minor:-0}"
if [ -z "$peak" ]; then
echo "Failed to parse peak memory for $ALLOCATOR" >&2
exit 1
fi
major="${major:-0}"
minor="${minor:-0}"
printf "%s\t%s\t%s\t%s\n" "$allocator" "$peak" "$major" "$minor" >> /work/alpine-results.tsv
done
printf "%s\t%s\t%s\t%s\n" "$ALLOCATOR" "$peak" "$major" "$minor" >> /work/alpine-results.tsv
INNER_EOF
)
@@ -58,10 +51,11 @@ docker run --rm \
-w /work \
-e TOKIO_FEATURE="$TOKIO_FEATURE" \
-e PLATFORM_LABEL="${PLATFORM_LABEL:-unknown}" \
-e ALLOCATOR="$ALLOCATOR" \
rust:alpine sh -lc "$ALPINE_SCRIPT"
{
echo "### ${PLATFORM_LABEL:-unknown} (tokio-multi-thread=${TOKIO_MULTI_THREAD:-false})"
echo "### ${PLATFORM_LABEL:-unknown} (tokio-multi-thread=${TOKIO_MULTI_THREAD:-false}, allocator=${ALLOCATOR})"
echo "Threads: $(nproc --all)"
echo ""
echo "| Allocator | Peak KB | Major PF | Minor PF |"
.github/scripts/allocator_bench_unix.sh
+42 -65
View File
@@ -6,88 +6,65 @@ if [[ "${TOKIO_MULTI_THREAD:-false}" == "true" ]]; then
TOKIO_FEATURE="tokio-multi-thread"
fi
allocators=(system jemalloc mimalloc_v2 mimalloc_v3)
if [[ "$RUNNER_OS" == "Linux" ]]; then
time_cmd=(/usr/bin/time -v)
parse_peak() { awk -F': ' '/Maximum resident set size/ {print $2; exit}'; }
parse_major() { awk -F': ' '/Major \(requiring I\/O\) page faults/ {print $2; exit}'; }
parse_minor() { awk -F': ' '/Minor \(reclaiming a frame\) page faults/ {print $2; exit}'; }
else
time_cmd=(/usr/bin/time -l)
parse_peak() { awk '/maximum resident set size/ {print $1; exit}'; }
parse_major() { awk '/page faults/ {print $1; exit}'; }
parse_minor() { awk '/page reclaims/ {print $1; exit}'; }
fi
build_features() {
local allocator="$1"
local features=""
if [[ "$allocator" != "system" ]]; then
features="$allocator"
fi
if [[ -n "$TOKIO_FEATURE" ]]; then
if [[ -n "$features" ]]; then
features="$features,$TOKIO_FEATURE"
else
features="$TOKIO_FEATURE"
fi
fi
echo "$features"
}
run_one() {
local allocator="$1"
local features
features="$(build_features "$allocator")"
local feature_args=()
allocator="$ALLOCATOR"
features=""
if [[ "$allocator" != "system" ]]; then
features="$allocator"
fi
if [[ -n "$TOKIO_FEATURE" ]]; then
if [[ -n "$features" ]]; then
feature_args=(--features "$features")
features="$features,$TOKIO_FEATURE"
else
features="$TOKIO_FEATURE"
fi
fi
cargo build --release --locked "${feature_args[@]}"
feature_args=()
if [[ -n "$features" ]]; then
feature_args=(--features "$features")
fi
local exe="target/release/proxy-scraper-checker"
if [[ "$RUNNER_OS" == "Windows" ]]; then
exe="target/release/proxy-scraper-checker.exe"
fi
local output
output="$(${time_cmd[@]} "$exe" 2>&1 >/dev/null)"
local peak
peak="$(echo "$output" | parse_peak)"
local major
major="$(echo "$output" | parse_major)"
local minor
minor="$(echo "$output" | parse_minor)"
if [[ -z "$peak" ]]; then
echo "Failed to parse peak memory for $allocator" >&2
exit 1
fi
major="${major:-0}"
minor="${minor:-0}"
if [[ "$RUNNER_OS" != "Linux" ]]; then
peak=$((peak / 1024))
fi
printf "%s\t%s\t%s\t%s\n" "$allocator" "$peak" "$major" "$minor" >> results.tsv
}
cargo build --release --locked "${feature_args[@]}"
exe="target/release/proxy-scraper-checker"
if [[ "$RUNNER_OS" == "Windows" ]]; then
allocators=(system mimalloc_v2 mimalloc_v3)
exe="target/release/proxy-scraper-checker.exe"
fi
output="$(${time_cmd[@]} "$exe" 2>&1 >/dev/null)"
if [[ "$RUNNER_OS" == "Linux" ]]; then
peak="$(echo "$output" | awk -F': ' '/Maximum resident set size/ {print $2; exit}')"
major="$(echo "$output" | awk -F': ' '/Major \(requiring I\/O\) page faults/ {print $2; exit}')"
minor="$(echo "$output" | awk -F': ' '/Minor \(reclaiming a frame\) page faults/ {print $2; exit}')"
else
peak="$(echo "$output" | awk '/maximum resident set size/ {print $1; exit}')"
major="$(echo "$output" | awk '/page faults/ {print $1; exit}')"
minor="$(echo "$output" | awk '/page reclaims/ {print $1; exit}')"
fi
if [[ -z "$peak" ]]; then
echo "Failed to parse peak memory for $allocator" >&2
exit 1
fi
major="${major:-0}"
minor="${minor:-0}"
if [[ "$RUNNER_OS" != "Linux" ]]; then
peak=$((peak / 1024))
fi
: > results.tsv
for allocator in "${allocators[@]}"; do
run_one "$allocator"
done
printf "%s\t%s\t%s\t%s\n" "$allocator" "$peak" "$major" "$minor" >> results.tsv
{
echo "### ${PLATFORM_LABEL:-unknown} (tokio-multi-thread=${TOKIO_MULTI_THREAD:-false})"
echo "### ${PLATFORM_LABEL:-unknown} (tokio-multi-thread=${TOKIO_MULTI_THREAD:-false}, allocator=${ALLOCATOR})"
if [[ "$RUNNER_OS" == "Linux" ]]; then
echo "Threads: $(nproc --all)"
elif [[ "$RUNNER_OS" == "macOS" ]]; then
.github/scripts/allocator_bench_windows.ps1
+29 -40
View File
@@ -1,52 +1,41 @@
$tokioOn = $env:TOKIO_MULTI_THREAD
$tokioFeature = if ($tokioOn -eq "true") { "tokio-multi-thread" } else { "" }
$allocators = @("system", "mimalloc_v2", "mimalloc_v3")
$allocator = $env:ALLOCATOR
$features = @()
if ($allocator -ne "system") { $features += $allocator }
if ($tokioFeature) { $features += $tokioFeature }
$featuresString = $features -join ","
function Build-Features([string]$allocator, [string]$tokio) {
$features = @()
if ($allocator -ne "system") { $features += $allocator }
if ($tokio) { $features += $tokio }
return ($features -join ",")
}
function Run-One([string]$allocator, [string]$features) {
$args = @("build", "--release", "--locked")
if ($features) { $args += "--features"; $args += $features }
$proc = Start-Process -FilePath "cargo" -ArgumentList $args -PassThru -NoNewWindow -Wait
$exe = "target\release\proxy-scraper-checker.exe"
$proc = Start-Process -FilePath $exe -PassThru -NoNewWindow
$peak = 0
$faults = 0
while (-not $proc.HasExited) {
Start-Sleep -Milliseconds 100
try {
$p = Get-Process -Id $proc.Id -ErrorAction Stop
$current = [math]::Max($p.WorkingSet64, $p.PeakWorkingSet64)
if ($current -gt $peak) { $peak = $current }
$wmi = Get-CimInstance Win32_Process -Filter "ProcessId=$($proc.Id)" -ErrorAction SilentlyContinue
if ($wmi -and $wmi.PageFaults -gt $faults) { $faults = $wmi.PageFaults }
} catch { }
}
$peakKb = [math]::Floor($peak / 1kb)
Add-Content -Path results.tsv -Value "$allocator`t$peakKb`t$faults"
$args = @("build", "--release", "--locked")
if ($featuresString) { $args += "--features"; $args += $featuresString }
Start-Process -FilePath "cargo" -ArgumentList $args -NoNewWindow -Wait
$exe = "target\release\proxy-scraper-checker.exe"
$proc = Start-Process -FilePath $exe -PassThru -NoNewWindow
$peak = 0
$faults = 0
while (-not $proc.HasExited) {
Start-Sleep -Milliseconds 100
try {
$p = Get-Process -Id $proc.Id -ErrorAction Stop
$current = [math]::Max($p.WorkingSet64, $p.PeakWorkingSet64)
if ($current -gt $peak) { $peak = $current }
$wmi = Get-CimInstance Win32_Process -Filter "ProcessId=$($proc.Id)" -ErrorAction SilentlyContinue
if ($wmi -and $wmi.PageFaults -gt $faults) { $faults = $wmi.PageFaults }
} catch { }
}
$peakKb = [math]::Floor($peak / 1kb)
if (Test-Path results.tsv) { Remove-Item results.tsv -Force }
foreach ($allocator in $allocators) {
$features = Build-Features $allocator $tokioFeature
Run-One $allocator $features
}
Add-Content -Path results.tsv -Value "$allocator`t$peakKb`t$faults"
$summary = @()
$summary += "### $($env:PLATFORM_LABEL) (tokio-multi-thread=$tokioOn)"
$summary += "### $($env:PLATFORM_LABEL) (tokio-multi-thread=$tokioOn, allocator=$allocator)"
$summary += "Threads: $($env:NUMBER_OF_PROCESSORS)"
$summary += ""
$summary += "| Allocator | Peak KB | Page Faults |"