Show progress of downloaded files in the terminal

This connects the stderr of download utilities like curl and wget to the
original stderr of the process so that they can detect a terminal and
print progress bars to it.
This commit is contained in:
Mislav Marohnić 2023-11-07 10:17:54 +01:00
parent 61e50df552
commit 93c50bbaf0
No known key found for this signature in database
5 changed files with 49 additions and 78 deletions

View file

@ -406,6 +406,7 @@ http() {
RUBY_BUILD_HTTP_CLIENT="${RUBY_BUILD_HTTP_CLIENT:-$(detect_http_client 2>&3)}" RUBY_BUILD_HTTP_CLIENT="${RUBY_BUILD_HTTP_CLIENT:-$(detect_http_client 2>&3)}"
[ -n "$RUBY_BUILD_HTTP_CLIENT" ] || return 1 [ -n "$RUBY_BUILD_HTTP_CLIENT" ] || return 1
# http_get_curl, http_get_wget, etc.
"http_${method}_${RUBY_BUILD_HTTP_CLIENT}" "$@" "http_${method}_${RUBY_BUILD_HTTP_CLIENT}" "$@"
} }
@ -423,37 +424,32 @@ detect_http_client() {
http_head_aria2c() { http_head_aria2c() {
# shellcheck disable=SC2086 # shellcheck disable=SC2086
aria2c --dry-run --no-conf=true ${ARIA2_OPTS} "$1" >&4 2>&1 aria2c --dry-run --no-conf=true $ARIA2_OPTS "$1" >/dev/null
} }
http_get_aria2c() { http_get_aria2c() {
local out="${2:-$(mktemp "out.XXXXXX")}"
# shellcheck disable=SC2086 # shellcheck disable=SC2086
if aria2c --allow-overwrite=true --no-conf=true -o "${out}" ${ARIA2_OPTS} "$1" >&4; then log_command aria2c --allow-overwrite=true --no-conf=true --console-log-level=warn --stderr $ARIA2_OPTS -o "$2" "$1" 2>&3
[ -n "$2" ] || cat "${out}"
else
false
fi
} }
http_head_curl() { http_head_curl() {
# shellcheck disable=SC2086 # shellcheck disable=SC2086
curl -qsILf ${CURL_OPTS} "$1" >&4 2>&1 curl -qsILf $CURL_OPTS "$1" >/dev/null
} }
http_get_curl() { http_get_curl() {
# shellcheck disable=SC2086 # shellcheck disable=SC2086
curl -q -o "${2:--}" -sSLf ${CURL_OPTS} "$1" log_command curl -q -fL $CURL_OPTS -o "$2" "$1" 2>&3
} }
http_head_wget() { http_head_wget() {
# shellcheck disable=SC2086 # shellcheck disable=SC2086
wget -q --spider ${WGET_OPTS} "$1" >&4 2>&1 wget -q --spider $WGET_OPTS "$1" >/dev/null
} }
http_get_wget() { http_get_wget() {
# shellcheck disable=SC2086 # shellcheck disable=SC2086
wget -nv ${WGET_OPTS} -O "${2:--}" "$1" log_command wget -nv --show-progress $WGET_OPTS -O "$2" "$1" 2>&3
} }
fetch_tarball() { fetch_tarball() {
@ -494,26 +490,21 @@ fetch_tarball() {
if ! reuse_existing_tarball "$package_filename" "$checksum"; then if ! reuse_existing_tarball "$package_filename" "$checksum"; then
local tarball_filename local tarball_filename
tarball_filename="$(basename "$package_url")" tarball_filename="$(basename "$package_url")"
echo "Downloading ${tarball_filename}..." >&2 log_info "Downloading ${tarball_filename}..."
# shellcheck disable=SC2015 # shellcheck disable=SC2015
http head "$mirror_url" && http head "$mirror_url" &&
download_tarball "$mirror_url" "$package_filename" "$checksum" || download_tarball "$mirror_url" "$package_filename" "$checksum" ||
download_tarball "$package_url" "$package_filename" "$checksum" download_tarball "$package_url" "$package_filename" "$checksum"
fi fi
{ if tar "$tar_args" "$package_filename"; then log_command tar "$tar_args" "$package_filename" >/dev/null
if [ ! -d "$package_name" ]; then
extracted_dir="$(find_extracted_directory)"
mv "$extracted_dir" "$package_name"
fi
if [ -z "$KEEP_BUILD_PATH" ]; then if [ ! -d "$package_name" ]; then
rm -f "$package_filename" extracted_dir="$(find_extracted_directory)"
else mv "$extracted_dir" "$package_name"
true fi
fi
fi [ -n "$KEEP_BUILD_PATH" ] || rm -f "$package_filename"
} >&4 2>&1
} }
find_extracted_directory() { find_extracted_directory() {
@ -541,8 +532,8 @@ reuse_existing_tarball() {
local cached_package_filename="${RUBY_BUILD_CACHE_PATH}/$package_filename" local cached_package_filename="${RUBY_BUILD_CACHE_PATH}/$package_filename"
[ -e "$cached_package_filename" ] || return 1 [ -e "$cached_package_filename" ] || return 1
verify_checksum "$cached_package_filename" "$checksum" >&4 2>&1 || return 1 verify_checksum "$cached_package_filename" "$checksum" || return 1
ln -s "$cached_package_filename" "$package_filename" >&4 2>&1 || return 1 ln -s "$cached_package_filename" "$package_filename" || return 1
} }
download_tarball() { download_tarball() {
@ -552,10 +543,8 @@ download_tarball() {
local package_filename="$2" local package_filename="$2"
local checksum="$3" local checksum="$3"
echo "-> $package_url" >&2 if http get "$package_url" "$package_filename"; then
verify_checksum "$package_filename" "$checksum" || return 1
if http get "$package_url" "$package_filename" >&4 2>&1; then
verify_checksum "$package_filename" "$checksum" >&4 2>&1 || return 1
else else
echo "error: failed to download $package_filename" >&2 echo "error: failed to download $package_filename" >&2
return 1 return 1
@ -563,9 +552,8 @@ download_tarball() {
if [ -n "$RUBY_BUILD_CACHE_PATH" ]; then if [ -n "$RUBY_BUILD_CACHE_PATH" ]; then
local cached_package_filename="${RUBY_BUILD_CACHE_PATH}/$package_filename" local cached_package_filename="${RUBY_BUILD_CACHE_PATH}/$package_filename"
{ mv "$package_filename" "$cached_package_filename" mv "$package_filename" "$cached_package_filename"
ln -s "$cached_package_filename" "$package_filename" ln -s "$cached_package_filename" "$package_filename"
} >&4 2>&1 || return 1
fi fi
} }

View file

@ -5,7 +5,7 @@ export RUBY_BUILD_SKIP_MIRROR=1
export RUBY_BUILD_CACHE_PATH="$TMP/cache" export RUBY_BUILD_CACHE_PATH="$TMP/cache"
@test "packages are saved to download cache" { @test "packages are saved to download cache" {
stub curl "-q -o * -*S* http://example.com/* : cp $FIXTURE_ROOT/\${5##*/} \$3" stub curl "-q -fL -o * http://example.com/* : cp $FIXTURE_ROOT/\${5##*/} \$4"
mkdir -p "$RUBY_BUILD_CACHE_PATH" mkdir -p "$RUBY_BUILD_CACHE_PATH"
install_fixture definitions/without-checksum install_fixture definitions/without-checksum
@ -56,7 +56,7 @@ export RUBY_BUILD_CACHE_PATH="$TMP/cache"
stub shasum true "echo invalid" "echo $checksum" stub shasum true "echo invalid" "echo $checksum"
stub curl "-*I* : true" \ stub curl "-*I* : true" \
"-q -o * -*S* https://?*/$checksum : cp $FIXTURE_ROOT/package-1.0.0.tar.gz \$3" "-q -fL -o * https://?*/$checksum : cp $FIXTURE_ROOT/package-1.0.0.tar.gz \$4"
mkdir -p "$RUBY_BUILD_CACHE_PATH" mkdir -p "$RUBY_BUILD_CACHE_PATH"
touch "${RUBY_BUILD_CACHE_PATH}/package-1.0.0.tar.gz" touch "${RUBY_BUILD_CACHE_PATH}/package-1.0.0.tar.gz"
@ -74,7 +74,7 @@ export RUBY_BUILD_CACHE_PATH="$TMP/cache"
@test "nonexistent cache directory is ignored" { @test "nonexistent cache directory is ignored" {
stub curl "-q -o * -*S* http://example.com/* : cp $FIXTURE_ROOT/\${5##*/} \$3" stub curl "-q -fL -o * http://example.com/* : cp $FIXTURE_ROOT/\${5##*/} \$4"
export RUBY_BUILD_CACHE_PATH="${TMP}/nonexistent" export RUBY_BUILD_CACHE_PATH="${TMP}/nonexistent"

View file

@ -6,7 +6,7 @@ export RUBY_BUILD_CACHE_PATH=
@test "package URL without checksum" { @test "package URL without checksum" {
stub curl "-q -o * -*S* http://example.com/* : cp $FIXTURE_ROOT/\${5##*/} \$3" stub curl "-q -fL -o * http://example.com/* : cp $FIXTURE_ROOT/\${5##*/} \$4"
install_fixture definitions/without-checksum install_fixture definitions/without-checksum
@ -19,7 +19,7 @@ export RUBY_BUILD_CACHE_PATH=
@test "package URL with valid checksum" { @test "package URL with valid checksum" {
stub shasum true "echo ba988b1bb4250dee0b9dd3d4d722f9c64b2bacfc805d1b6eba7426bda72dd3c5" stub shasum true "echo ba988b1bb4250dee0b9dd3d4d722f9c64b2bacfc805d1b6eba7426bda72dd3c5"
stub curl "-q -o * -*S* http://example.com/* : cp $FIXTURE_ROOT/\${5##*/} \$3" stub curl "-q -fL -o * http://example.com/* : cp $FIXTURE_ROOT/\${5##*/} \$4"
install_fixture definitions/with-checksum install_fixture definitions/with-checksum
@ -33,7 +33,7 @@ export RUBY_BUILD_CACHE_PATH=
@test "package URL with invalid checksum" { @test "package URL with invalid checksum" {
stub shasum true "echo ba988b1bb4250dee0b9dd3d4d722f9c64b2bacfc805d1b6eba7426bda72dd3c5" stub shasum true "echo ba988b1bb4250dee0b9dd3d4d722f9c64b2bacfc805d1b6eba7426bda72dd3c5"
stub curl "-q -o * -*S* http://example.com/* : cp $FIXTURE_ROOT/\${5##*/} \$3" stub curl "-q -fL -o * http://example.com/* : cp $FIXTURE_ROOT/\${5##*/} \$4"
install_fixture definitions/with-invalid-checksum install_fixture definitions/with-invalid-checksum
@ -47,7 +47,7 @@ export RUBY_BUILD_CACHE_PATH=
@test "package URL with checksum but no shasum support" { @test "package URL with checksum but no shasum support" {
stub shasum false stub shasum false
stub curl "-q -o * -*S* http://example.com/* : cp $FIXTURE_ROOT/\${5##*/} \$3" stub curl "-q -fL -o * http://example.com/* : cp $FIXTURE_ROOT/\${5##*/} \$4"
install_fixture definitions/with-checksum install_fixture definitions/with-checksum
@ -61,7 +61,7 @@ export RUBY_BUILD_CACHE_PATH=
@test "package URL with valid md5 checksum" { @test "package URL with valid md5 checksum" {
stub md5 true "echo 83e6d7725e20166024a1eb74cde80677" stub md5 true "echo 83e6d7725e20166024a1eb74cde80677"
stub curl "-q -o * -*S* http://example.com/* : cp $FIXTURE_ROOT/\${5##*/} \$3" stub curl "-q -fL -o * http://example.com/* : cp $FIXTURE_ROOT/\${5##*/} \$4"
install_fixture definitions/with-md5-checksum install_fixture definitions/with-md5-checksum
@ -75,7 +75,7 @@ export RUBY_BUILD_CACHE_PATH=
@test "package URL with md5 checksum but no md5 support" { @test "package URL with md5 checksum but no md5 support" {
stub md5 false stub md5 false
stub curl "-q -o * -*S* http://example.com/* : cp $FIXTURE_ROOT/\${5##*/} \$3" stub curl "-q -fL -o * http://example.com/* : cp $FIXTURE_ROOT/\${5##*/} \$4"
install_fixture definitions/with-md5-checksum install_fixture definitions/with-md5-checksum
@ -89,7 +89,7 @@ export RUBY_BUILD_CACHE_PATH=
@test "package with invalid checksum" { @test "package with invalid checksum" {
stub shasum true "echo invalid" stub shasum true "echo invalid"
stub curl "-q -o * -*S* http://example.com/* : cp $FIXTURE_ROOT/\${5##*/} \$3" stub curl "-q -fL -o * http://example.com/* : cp $FIXTURE_ROOT/\${5##*/} \$4"
install_fixture definitions/with-checksum install_fixture definitions/with-checksum
@ -125,7 +125,7 @@ DEF
stub shasum true \ stub shasum true \
"echo invalid" \ "echo invalid" \
"echo ba988b1bb4250dee0b9dd3d4d722f9c64b2bacfc805d1b6eba7426bda72dd3c5" "echo ba988b1bb4250dee0b9dd3d4d722f9c64b2bacfc805d1b6eba7426bda72dd3c5"
stub curl "-q -o * -*S* http://example.com/* : cp $FIXTURE_ROOT/\${5##*/} \$3" stub curl "-q -fL -o * http://example.com/* : cp $FIXTURE_ROOT/\${5##*/} \$4"
export -n RUBY_BUILD_CACHE_PATH export -n RUBY_BUILD_CACHE_PATH
export RUBY_BUILD_BUILD_PATH="${TMP}/build" export RUBY_BUILD_BUILD_PATH="${TMP}/build"
@ -144,7 +144,7 @@ DEF
} }
@test "package URL with checksum of unexpected length" { @test "package URL with checksum of unexpected length" {
stub curl "-q -o * -*S* http://example.com/* : cp $FIXTURE_ROOT/\${5##*/} \$3" stub curl "-q -fL -o * http://example.com/* : cp $FIXTURE_ROOT/\${5##*/} \$4"
run_inline_definition <<DEF run_inline_definition <<DEF
install_package "package-1.0.0" "http://example.com/packages/package-1.0.0.tar.gz#checksum_of_unexpected_length" copy install_package "package-1.0.0" "http://example.com/packages/package-1.0.0.tar.gz#checksum_of_unexpected_length" copy

View file

@ -14,7 +14,6 @@ setup() {
install_fixture definitions/without-checksum install_fixture definitions/without-checksum
assert_failure assert_failure
assert_output_contains "> http://example.com/packages/package-1.0.0.tar.gz"
assert_output_contains "error: failed to download package-1.0.0.tar.gz" assert_output_contains "error: failed to download package-1.0.0.tar.gz"
} }
@ -30,16 +29,11 @@ setup() {
@test "using aria2c if available" { @test "using aria2c if available" {
export RUBY_BUILD_ARIA2_OPTS= export RUBY_BUILD_ARIA2_OPTS=
export -n RUBY_BUILD_HTTP_CLIENT export -n RUBY_BUILD_HTTP_CLIENT
stub aria2c "--allow-overwrite=true --no-conf=true -o * http://example.com/* : cp $FIXTURE_ROOT/\${5##*/} \$4" stub aria2c "--allow-overwrite=true --no-conf=true --console-log-level=warn --stderr -o * http://example.com/* : cp $FIXTURE_ROOT/\${7##*/} \$6"
install_fixture definitions/without-checksum install_fixture definitions/without-checksum
assert_success assert_success
assert_output <<OUT assert_output_contains "Downloading package-1.0.0.tar.gz..."
Downloading package-1.0.0.tar.gz...
-> http://example.com/packages/package-1.0.0.tar.gz
Installing package-1.0.0...
Installed package-1.0.0 to ${TMP}/install
OUT
unstub aria2c unstub aria2c
} }
@ -50,11 +44,7 @@ OUT
install_git "package-dev" "http://example.com/packages/package.git" master copy install_git "package-dev" "http://example.com/packages/package.git" master copy
DEF DEF
assert_success assert_success
assert_output <<OUT assert_output_contains "Cloning http://example.com/packages/package.git..."
Cloning http://example.com/packages/package.git...
Installing package-dev...
Installed package-dev to ${TMP}/install
OUT
unstub git unstub git
} }
@ -68,10 +58,6 @@ OUT
install_git "package-dev" "http://example.com/packages/package.git" master copy install_git "package-dev" "http://example.com/packages/package.git" master copy
DEF DEF
assert_success assert_success
assert_output <<OUT assert_output_contains "Cloning http://example.com/packages/package.git..."
Cloning http://example.com/packages/package.git...
Installing package-dev...
Installed package-dev to ${TMP}/install
OUT
unstub git unstub git
} }

View file

@ -8,10 +8,9 @@ export RUBY_BUILD_MIRROR_URL=http://mirror.example.com
@test "package URL without checksum bypasses mirror" { @test "package URL without checksum bypasses mirror" {
stub shasum true stub shasum true
stub curl "-q -o * -*S* http://example.com/* : cp $FIXTURE_ROOT/\${5##*/} \$3" stub curl "-q -fL -o * http://example.com/* : cp $FIXTURE_ROOT/\${5##*/} \$4"
install_fixture definitions/without-checksum install_fixture definitions/without-checksum
echo "$output" >&2
assert_success assert_success
assert [ -x "${INSTALL_ROOT}/bin/package" ] assert [ -x "${INSTALL_ROOT}/bin/package" ]
@ -23,7 +22,7 @@ export RUBY_BUILD_MIRROR_URL=http://mirror.example.com
@test "package URL with checksum but no shasum support bypasses mirror" { @test "package URL with checksum but no shasum support bypasses mirror" {
stub shasum false stub shasum false
stub curl "-q -o * -*S* http://example.com/* : cp $FIXTURE_ROOT/\${5##*/} \$3" stub curl "-q -fL -o * http://example.com/* : cp $FIXTURE_ROOT/\${5##*/} \$4"
install_fixture definitions/with-checksum install_fixture definitions/with-checksum
@ -41,7 +40,7 @@ export RUBY_BUILD_MIRROR_URL=http://mirror.example.com
stub shasum true "echo $checksum" stub shasum true "echo $checksum"
stub curl "-*I* $mirror_url : true" \ stub curl "-*I* $mirror_url : true" \
"-q -o * -*S* $mirror_url : cp $FIXTURE_ROOT/package-1.0.0.tar.gz \$3" "-q -fL -o * $mirror_url : cp $FIXTURE_ROOT/package-1.0.0.tar.gz \$4"
install_fixture definitions/with-checksum install_fixture definitions/with-checksum
@ -59,7 +58,7 @@ export RUBY_BUILD_MIRROR_URL=http://mirror.example.com
stub shasum true "echo $checksum" stub shasum true "echo $checksum"
stub curl "-*I* $mirror_url : false" \ stub curl "-*I* $mirror_url : false" \
"-q -o * -*S* http://example.com/* : cp $FIXTURE_ROOT/\${5##*/} \$3" "-q -fL -o * http://example.com/* : cp $FIXTURE_ROOT/\${5##*/} \$4"
install_fixture definitions/with-checksum install_fixture definitions/with-checksum
@ -77,11 +76,10 @@ export RUBY_BUILD_MIRROR_URL=http://mirror.example.com
stub shasum true "echo invalid" "echo $checksum" stub shasum true "echo invalid" "echo $checksum"
stub curl "-*I* $mirror_url : true" \ stub curl "-*I* $mirror_url : true" \
"-q -o * -*S* $mirror_url : cp $FIXTURE_ROOT/package-1.0.0.tar.gz \$3" \ "-q -fL -o * $mirror_url : cp $FIXTURE_ROOT/package-1.0.0.tar.gz \$4" \
"-q -o * -*S* http://example.com/* : cp $FIXTURE_ROOT/\${5##*/} \$3" "-q -fL -o * http://example.com/* : cp $FIXTURE_ROOT/\${5##*/} \$4"
install_fixture definitions/with-checksum install_fixture definitions/with-checksum
echo "$output" >&2
assert_success assert_success
assert [ -x "${INSTALL_ROOT}/bin/package" ] assert [ -x "${INSTALL_ROOT}/bin/package" ]
@ -97,7 +95,7 @@ export RUBY_BUILD_MIRROR_URL=http://mirror.example.com
stub shasum true "echo $checksum" stub shasum true "echo $checksum"
stub curl "-*I* : true" \ stub curl "-*I* : true" \
"-q -o * -*S* https://?*/$checksum : cp $FIXTURE_ROOT/package-1.0.0.tar.gz \$3" \ "-q -fL -o * https://?*/$checksum : cp $FIXTURE_ROOT/package-1.0.0.tar.gz \$4" \
install_fixture definitions/with-checksum install_fixture definitions/with-checksum
@ -114,7 +112,7 @@ export RUBY_BUILD_MIRROR_URL=http://mirror.example.com
local checksum="ba988b1bb4250dee0b9dd3d4d722f9c64b2bacfc805d1b6eba7426bda72dd3c5" local checksum="ba988b1bb4250dee0b9dd3d4d722f9c64b2bacfc805d1b6eba7426bda72dd3c5"
stub shasum true "echo $checksum" stub shasum true "echo $checksum"
stub curl "-q -o * -*S* https://cache.ruby-lang.org/* : cp $FIXTURE_ROOT/\${5##*/} \$3" stub curl "-q -fL -o * https://cache.ruby-lang.org/* : cp $FIXTURE_ROOT/\${5##*/} \$4"
run_inline_definition <<DEF run_inline_definition <<DEF
install_package "package-1.0.0" "https://cache.ruby-lang.org/packages/package-1.0.0.tar.gz#ba988b1bb4250dee0b9dd3d4d722f9c64b2bacfc805d1b6eba7426bda72dd3c5" copy install_package "package-1.0.0" "https://cache.ruby-lang.org/packages/package-1.0.0.tar.gz#ba988b1bb4250dee0b9dd3d4d722f9c64b2bacfc805d1b6eba7426bda72dd3c5" copy
@ -135,7 +133,7 @@ DEF
stub shasum true "echo $checksum" stub shasum true "echo $checksum"
stub curl "-*I* $RUBY_BUILD_MIRROR_PACKAGE_URL : true" \ stub curl "-*I* $RUBY_BUILD_MIRROR_PACKAGE_URL : true" \
"-q -o * -*S* $RUBY_BUILD_MIRROR_PACKAGE_URL : cp $FIXTURE_ROOT/package-1.0.0.tar.gz \$3" "-q -fL -o * $RUBY_BUILD_MIRROR_PACKAGE_URL : cp $FIXTURE_ROOT/package-1.0.0.tar.gz \$4"
install_fixture definitions/with-checksum install_fixture definitions/with-checksum
@ -154,7 +152,7 @@ DEF
stub shasum true "echo $checksum" stub shasum true "echo $checksum"
stub curl "-*I* $RUBY_BUILD_MIRROR_PACKAGE_URL : false" \ stub curl "-*I* $RUBY_BUILD_MIRROR_PACKAGE_URL : false" \
"-q -o * -*S* http://example.com/* : cp $FIXTURE_ROOT/\${5##*/} \$3" "-q -fL -o * http://example.com/* : cp $FIXTURE_ROOT/\${5##*/} \$4"
install_fixture definitions/with-checksum install_fixture definitions/with-checksum
@ -173,11 +171,10 @@ DEF
stub shasum true "echo invalid" "echo $checksum" stub shasum true "echo invalid" "echo $checksum"
stub curl "-*I* $RUBY_BUILD_MIRROR_PACKAGE_URL : true" \ stub curl "-*I* $RUBY_BUILD_MIRROR_PACKAGE_URL : true" \
"-q -o * -*S* $RUBY_BUILD_MIRROR_PACKAGE_URL : cp $FIXTURE_ROOT/package-1.0.0.tar.gz \$3" \ "-q -fL -o * $RUBY_BUILD_MIRROR_PACKAGE_URL : cp $FIXTURE_ROOT/package-1.0.0.tar.gz \$4" \
"-q -o * -*S* http://example.com/* : cp $FIXTURE_ROOT/\${5##*/} \$3" "-q -fL -o * http://example.com/* : cp $FIXTURE_ROOT/\${5##*/} \$4"
install_fixture definitions/with-checksum install_fixture definitions/with-checksum
echo "$output" >&2
assert_success assert_success
assert [ -x "${INSTALL_ROOT}/bin/package" ] assert [ -x "${INSTALL_ROOT}/bin/package" ]