From 93c50bbaf069f8184b98d6023aed92baf73aadf5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mislav=20Marohni=C4=87?= Date: Tue, 7 Nov 2023 10:17:54 +0100 Subject: [PATCH] Show progress of downloaded files in the terminal This connects the stderr of download utilities like curl and wget to the original stderr of the process so that they can detect a terminal and print progress bars to it. --- bin/ruby-build | 54 ++++++++++++++++++---------------------------- test/cache.bats | 6 +++--- test/checksum.bats | 18 ++++++++-------- test/fetch.bats | 22 ++++--------------- test/mirror.bats | 27 +++++++++++------------ 5 files changed, 49 insertions(+), 78 deletions(-) diff --git a/bin/ruby-build b/bin/ruby-build index 791184b0..12fbb116 100755 --- a/bin/ruby-build +++ b/bin/ruby-build @@ -406,6 +406,7 @@ http() { RUBY_BUILD_HTTP_CLIENT="${RUBY_BUILD_HTTP_CLIENT:-$(detect_http_client 2>&3)}" [ -n "$RUBY_BUILD_HTTP_CLIENT" ] || return 1 + # http_get_curl, http_get_wget, etc. "http_${method}_${RUBY_BUILD_HTTP_CLIENT}" "$@" } @@ -423,37 +424,32 @@ detect_http_client() { http_head_aria2c() { # shellcheck disable=SC2086 - aria2c --dry-run --no-conf=true ${ARIA2_OPTS} "$1" >&4 2>&1 + aria2c --dry-run --no-conf=true $ARIA2_OPTS "$1" >/dev/null } http_get_aria2c() { - local out="${2:-$(mktemp "out.XXXXXX")}" # shellcheck disable=SC2086 - if aria2c --allow-overwrite=true --no-conf=true -o "${out}" ${ARIA2_OPTS} "$1" >&4; then - [ -n "$2" ] || cat "${out}" - else - false - fi + log_command aria2c --allow-overwrite=true --no-conf=true --console-log-level=warn --stderr $ARIA2_OPTS -o "$2" "$1" 2>&3 } http_head_curl() { # shellcheck disable=SC2086 - curl -qsILf ${CURL_OPTS} "$1" >&4 2>&1 + curl -qsILf $CURL_OPTS "$1" >/dev/null } http_get_curl() { # shellcheck disable=SC2086 - curl -q -o "${2:--}" -sSLf ${CURL_OPTS} "$1" + log_command curl -q -fL $CURL_OPTS -o "$2" "$1" 2>&3 } http_head_wget() { # shellcheck disable=SC2086 - wget -q --spider ${WGET_OPTS} "$1" >&4 2>&1 + wget -q --spider $WGET_OPTS "$1" >/dev/null } http_get_wget() { # shellcheck disable=SC2086 - wget -nv ${WGET_OPTS} -O "${2:--}" "$1" + log_command wget -nv --show-progress $WGET_OPTS -O "$2" "$1" 2>&3 } fetch_tarball() { @@ -494,26 +490,21 @@ fetch_tarball() { if ! reuse_existing_tarball "$package_filename" "$checksum"; then local tarball_filename tarball_filename="$(basename "$package_url")" - echo "Downloading ${tarball_filename}..." >&2 + log_info "Downloading ${tarball_filename}..." # shellcheck disable=SC2015 http head "$mirror_url" && download_tarball "$mirror_url" "$package_filename" "$checksum" || download_tarball "$package_url" "$package_filename" "$checksum" fi - { if tar "$tar_args" "$package_filename"; then - if [ ! -d "$package_name" ]; then - extracted_dir="$(find_extracted_directory)" - mv "$extracted_dir" "$package_name" - fi + log_command tar "$tar_args" "$package_filename" >/dev/null - if [ -z "$KEEP_BUILD_PATH" ]; then - rm -f "$package_filename" - else - true - fi - fi - } >&4 2>&1 + if [ ! -d "$package_name" ]; then + extracted_dir="$(find_extracted_directory)" + mv "$extracted_dir" "$package_name" + fi + + [ -n "$KEEP_BUILD_PATH" ] || rm -f "$package_filename" } find_extracted_directory() { @@ -541,8 +532,8 @@ reuse_existing_tarball() { local cached_package_filename="${RUBY_BUILD_CACHE_PATH}/$package_filename" [ -e "$cached_package_filename" ] || return 1 - verify_checksum "$cached_package_filename" "$checksum" >&4 2>&1 || return 1 - ln -s "$cached_package_filename" "$package_filename" >&4 2>&1 || return 1 + verify_checksum "$cached_package_filename" "$checksum" || return 1 + ln -s "$cached_package_filename" "$package_filename" || return 1 } download_tarball() { @@ -552,10 +543,8 @@ download_tarball() { local package_filename="$2" local checksum="$3" - echo "-> $package_url" >&2 - - if http get "$package_url" "$package_filename" >&4 2>&1; then - verify_checksum "$package_filename" "$checksum" >&4 2>&1 || return 1 + if http get "$package_url" "$package_filename"; then + verify_checksum "$package_filename" "$checksum" || return 1 else echo "error: failed to download $package_filename" >&2 return 1 @@ -563,9 +552,8 @@ download_tarball() { if [ -n "$RUBY_BUILD_CACHE_PATH" ]; then local cached_package_filename="${RUBY_BUILD_CACHE_PATH}/$package_filename" - { mv "$package_filename" "$cached_package_filename" - ln -s "$cached_package_filename" "$package_filename" - } >&4 2>&1 || return 1 + mv "$package_filename" "$cached_package_filename" + ln -s "$cached_package_filename" "$package_filename" fi } diff --git a/test/cache.bats b/test/cache.bats index b99dbcdb..e43047c6 100644 --- a/test/cache.bats +++ b/test/cache.bats @@ -5,7 +5,7 @@ export RUBY_BUILD_SKIP_MIRROR=1 export RUBY_BUILD_CACHE_PATH="$TMP/cache" @test "packages are saved to download cache" { - stub curl "-q -o * -*S* http://example.com/* : cp $FIXTURE_ROOT/\${5##*/} \$3" + stub curl "-q -fL -o * http://example.com/* : cp $FIXTURE_ROOT/\${5##*/} \$4" mkdir -p "$RUBY_BUILD_CACHE_PATH" install_fixture definitions/without-checksum @@ -56,7 +56,7 @@ export RUBY_BUILD_CACHE_PATH="$TMP/cache" stub shasum true "echo invalid" "echo $checksum" stub curl "-*I* : true" \ - "-q -o * -*S* https://?*/$checksum : cp $FIXTURE_ROOT/package-1.0.0.tar.gz \$3" + "-q -fL -o * https://?*/$checksum : cp $FIXTURE_ROOT/package-1.0.0.tar.gz \$4" mkdir -p "$RUBY_BUILD_CACHE_PATH" touch "${RUBY_BUILD_CACHE_PATH}/package-1.0.0.tar.gz" @@ -74,7 +74,7 @@ export RUBY_BUILD_CACHE_PATH="$TMP/cache" @test "nonexistent cache directory is ignored" { - stub curl "-q -o * -*S* http://example.com/* : cp $FIXTURE_ROOT/\${5##*/} \$3" + stub curl "-q -fL -o * http://example.com/* : cp $FIXTURE_ROOT/\${5##*/} \$4" export RUBY_BUILD_CACHE_PATH="${TMP}/nonexistent" diff --git a/test/checksum.bats b/test/checksum.bats index d3130f6a..3c60ca1f 100644 --- a/test/checksum.bats +++ b/test/checksum.bats @@ -6,7 +6,7 @@ export RUBY_BUILD_CACHE_PATH= @test "package URL without checksum" { - stub curl "-q -o * -*S* http://example.com/* : cp $FIXTURE_ROOT/\${5##*/} \$3" + stub curl "-q -fL -o * http://example.com/* : cp $FIXTURE_ROOT/\${5##*/} \$4" install_fixture definitions/without-checksum @@ -19,7 +19,7 @@ export RUBY_BUILD_CACHE_PATH= @test "package URL with valid checksum" { stub shasum true "echo ba988b1bb4250dee0b9dd3d4d722f9c64b2bacfc805d1b6eba7426bda72dd3c5" - stub curl "-q -o * -*S* http://example.com/* : cp $FIXTURE_ROOT/\${5##*/} \$3" + stub curl "-q -fL -o * http://example.com/* : cp $FIXTURE_ROOT/\${5##*/} \$4" install_fixture definitions/with-checksum @@ -33,7 +33,7 @@ export RUBY_BUILD_CACHE_PATH= @test "package URL with invalid checksum" { stub shasum true "echo ba988b1bb4250dee0b9dd3d4d722f9c64b2bacfc805d1b6eba7426bda72dd3c5" - stub curl "-q -o * -*S* http://example.com/* : cp $FIXTURE_ROOT/\${5##*/} \$3" + stub curl "-q -fL -o * http://example.com/* : cp $FIXTURE_ROOT/\${5##*/} \$4" install_fixture definitions/with-invalid-checksum @@ -47,7 +47,7 @@ export RUBY_BUILD_CACHE_PATH= @test "package URL with checksum but no shasum support" { stub shasum false - stub curl "-q -o * -*S* http://example.com/* : cp $FIXTURE_ROOT/\${5##*/} \$3" + stub curl "-q -fL -o * http://example.com/* : cp $FIXTURE_ROOT/\${5##*/} \$4" install_fixture definitions/with-checksum @@ -61,7 +61,7 @@ export RUBY_BUILD_CACHE_PATH= @test "package URL with valid md5 checksum" { stub md5 true "echo 83e6d7725e20166024a1eb74cde80677" - stub curl "-q -o * -*S* http://example.com/* : cp $FIXTURE_ROOT/\${5##*/} \$3" + stub curl "-q -fL -o * http://example.com/* : cp $FIXTURE_ROOT/\${5##*/} \$4" install_fixture definitions/with-md5-checksum @@ -75,7 +75,7 @@ export RUBY_BUILD_CACHE_PATH= @test "package URL with md5 checksum but no md5 support" { stub md5 false - stub curl "-q -o * -*S* http://example.com/* : cp $FIXTURE_ROOT/\${5##*/} \$3" + stub curl "-q -fL -o * http://example.com/* : cp $FIXTURE_ROOT/\${5##*/} \$4" install_fixture definitions/with-md5-checksum @@ -89,7 +89,7 @@ export RUBY_BUILD_CACHE_PATH= @test "package with invalid checksum" { stub shasum true "echo invalid" - stub curl "-q -o * -*S* http://example.com/* : cp $FIXTURE_ROOT/\${5##*/} \$3" + stub curl "-q -fL -o * http://example.com/* : cp $FIXTURE_ROOT/\${5##*/} \$4" install_fixture definitions/with-checksum @@ -125,7 +125,7 @@ DEF stub shasum true \ "echo invalid" \ "echo ba988b1bb4250dee0b9dd3d4d722f9c64b2bacfc805d1b6eba7426bda72dd3c5" - stub curl "-q -o * -*S* http://example.com/* : cp $FIXTURE_ROOT/\${5##*/} \$3" + stub curl "-q -fL -o * http://example.com/* : cp $FIXTURE_ROOT/\${5##*/} \$4" export -n RUBY_BUILD_CACHE_PATH export RUBY_BUILD_BUILD_PATH="${TMP}/build" @@ -144,7 +144,7 @@ DEF } @test "package URL with checksum of unexpected length" { - stub curl "-q -o * -*S* http://example.com/* : cp $FIXTURE_ROOT/\${5##*/} \$3" + stub curl "-q -fL -o * http://example.com/* : cp $FIXTURE_ROOT/\${5##*/} \$4" run_inline_definition < http://example.com/packages/package-1.0.0.tar.gz" assert_output_contains "error: failed to download package-1.0.0.tar.gz" } @@ -30,16 +29,11 @@ setup() { @test "using aria2c if available" { export RUBY_BUILD_ARIA2_OPTS= export -n RUBY_BUILD_HTTP_CLIENT - stub aria2c "--allow-overwrite=true --no-conf=true -o * http://example.com/* : cp $FIXTURE_ROOT/\${5##*/} \$4" + stub aria2c "--allow-overwrite=true --no-conf=true --console-log-level=warn --stderr -o * http://example.com/* : cp $FIXTURE_ROOT/\${7##*/} \$6" install_fixture definitions/without-checksum assert_success - assert_output < http://example.com/packages/package-1.0.0.tar.gz -Installing package-1.0.0... -Installed package-1.0.0 to ${TMP}/install -OUT + assert_output_contains "Downloading package-1.0.0.tar.gz..." unstub aria2c } @@ -50,11 +44,7 @@ OUT install_git "package-dev" "http://example.com/packages/package.git" master copy DEF assert_success - assert_output <&2 assert_success assert [ -x "${INSTALL_ROOT}/bin/package" ] @@ -23,7 +22,7 @@ export RUBY_BUILD_MIRROR_URL=http://mirror.example.com @test "package URL with checksum but no shasum support bypasses mirror" { stub shasum false - stub curl "-q -o * -*S* http://example.com/* : cp $FIXTURE_ROOT/\${5##*/} \$3" + stub curl "-q -fL -o * http://example.com/* : cp $FIXTURE_ROOT/\${5##*/} \$4" install_fixture definitions/with-checksum @@ -41,7 +40,7 @@ export RUBY_BUILD_MIRROR_URL=http://mirror.example.com stub shasum true "echo $checksum" stub curl "-*I* $mirror_url : true" \ - "-q -o * -*S* $mirror_url : cp $FIXTURE_ROOT/package-1.0.0.tar.gz \$3" + "-q -fL -o * $mirror_url : cp $FIXTURE_ROOT/package-1.0.0.tar.gz \$4" install_fixture definitions/with-checksum @@ -59,7 +58,7 @@ export RUBY_BUILD_MIRROR_URL=http://mirror.example.com stub shasum true "echo $checksum" stub curl "-*I* $mirror_url : false" \ - "-q -o * -*S* http://example.com/* : cp $FIXTURE_ROOT/\${5##*/} \$3" + "-q -fL -o * http://example.com/* : cp $FIXTURE_ROOT/\${5##*/} \$4" install_fixture definitions/with-checksum @@ -77,11 +76,10 @@ export RUBY_BUILD_MIRROR_URL=http://mirror.example.com stub shasum true "echo invalid" "echo $checksum" stub curl "-*I* $mirror_url : true" \ - "-q -o * -*S* $mirror_url : cp $FIXTURE_ROOT/package-1.0.0.tar.gz \$3" \ - "-q -o * -*S* http://example.com/* : cp $FIXTURE_ROOT/\${5##*/} \$3" + "-q -fL -o * $mirror_url : cp $FIXTURE_ROOT/package-1.0.0.tar.gz \$4" \ + "-q -fL -o * http://example.com/* : cp $FIXTURE_ROOT/\${5##*/} \$4" install_fixture definitions/with-checksum - echo "$output" >&2 assert_success assert [ -x "${INSTALL_ROOT}/bin/package" ] @@ -97,7 +95,7 @@ export RUBY_BUILD_MIRROR_URL=http://mirror.example.com stub shasum true "echo $checksum" stub curl "-*I* : true" \ - "-q -o * -*S* https://?*/$checksum : cp $FIXTURE_ROOT/package-1.0.0.tar.gz \$3" \ + "-q -fL -o * https://?*/$checksum : cp $FIXTURE_ROOT/package-1.0.0.tar.gz \$4" \ install_fixture definitions/with-checksum @@ -114,7 +112,7 @@ export RUBY_BUILD_MIRROR_URL=http://mirror.example.com local checksum="ba988b1bb4250dee0b9dd3d4d722f9c64b2bacfc805d1b6eba7426bda72dd3c5" stub shasum true "echo $checksum" - stub curl "-q -o * -*S* https://cache.ruby-lang.org/* : cp $FIXTURE_ROOT/\${5##*/} \$3" + stub curl "-q -fL -o * https://cache.ruby-lang.org/* : cp $FIXTURE_ROOT/\${5##*/} \$4" run_inline_definition <&2 assert_success assert [ -x "${INSTALL_ROOT}/bin/package" ]