From 93c50bbaf069f8184b98d6023aed92baf73aadf5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mislav=20Marohni=C4=87?= <git@mislav.net>
Date: Tue, 7 Nov 2023 10:17:54 +0100
Subject: [PATCH] Show progress of downloaded files in the terminal

This connects the stderr of download utilities like curl and wget to the
original stderr of the process so that they can detect a terminal and
print progress bars to it.
---
 bin/ruby-build     | 54 ++++++++++++++++++----------------------------
 test/cache.bats    |  6 +++---
 test/checksum.bats | 18 ++++++++--------
 test/fetch.bats    | 22 ++++---------------
 test/mirror.bats   | 27 +++++++++++------------
 5 files changed, 49 insertions(+), 78 deletions(-)

diff --git a/bin/ruby-build b/bin/ruby-build
index 791184b0..12fbb116 100755
--- a/bin/ruby-build
+++ b/bin/ruby-build
@@ -406,6 +406,7 @@ http() {
   RUBY_BUILD_HTTP_CLIENT="${RUBY_BUILD_HTTP_CLIENT:-$(detect_http_client 2>&3)}"
   [ -n "$RUBY_BUILD_HTTP_CLIENT" ] || return 1
 
+  # http_get_curl, http_get_wget, etc.
   "http_${method}_${RUBY_BUILD_HTTP_CLIENT}" "$@"
 }
 
@@ -423,37 +424,32 @@ detect_http_client() {
 
 http_head_aria2c() {
   # shellcheck disable=SC2086
-  aria2c --dry-run --no-conf=true ${ARIA2_OPTS} "$1" >&4 2>&1
+  aria2c --dry-run --no-conf=true $ARIA2_OPTS "$1" >/dev/null
 }
 
 http_get_aria2c() {
-  local out="${2:-$(mktemp "out.XXXXXX")}"
   # shellcheck disable=SC2086
-  if aria2c --allow-overwrite=true --no-conf=true -o "${out}" ${ARIA2_OPTS} "$1" >&4; then
-    [ -n "$2" ] || cat "${out}"
-  else
-    false
-  fi
+  log_command aria2c --allow-overwrite=true --no-conf=true --console-log-level=warn --stderr $ARIA2_OPTS -o "$2" "$1" 2>&3
 }
 
 http_head_curl() {
   # shellcheck disable=SC2086
-  curl -qsILf ${CURL_OPTS} "$1" >&4 2>&1
+  curl -qsILf $CURL_OPTS "$1" >/dev/null
 }
 
 http_get_curl() {
   # shellcheck disable=SC2086
-  curl -q -o "${2:--}" -sSLf ${CURL_OPTS} "$1"
+  log_command curl -q -fL $CURL_OPTS -o "$2" "$1" 2>&3
 }
 
 http_head_wget() {
   # shellcheck disable=SC2086
-  wget -q --spider ${WGET_OPTS} "$1" >&4 2>&1
+  wget -q --spider $WGET_OPTS "$1" >/dev/null
 }
 
 http_get_wget() {
   # shellcheck disable=SC2086
-  wget -nv ${WGET_OPTS} -O "${2:--}" "$1"
+  log_command wget -nv --show-progress $WGET_OPTS -O "$2" "$1" 2>&3
 }
 
 fetch_tarball() {
@@ -494,26 +490,21 @@ fetch_tarball() {
   if ! reuse_existing_tarball "$package_filename" "$checksum"; then
     local tarball_filename
     tarball_filename="$(basename "$package_url")"
-    echo "Downloading ${tarball_filename}..." >&2
+    log_info "Downloading ${tarball_filename}..."
     # shellcheck disable=SC2015
     http head "$mirror_url" &&
     download_tarball "$mirror_url" "$package_filename" "$checksum" ||
     download_tarball "$package_url" "$package_filename" "$checksum"
   fi
 
-  { if tar "$tar_args" "$package_filename"; then
-      if [ ! -d "$package_name" ]; then
-        extracted_dir="$(find_extracted_directory)"
-        mv "$extracted_dir" "$package_name"
-      fi
+  log_command tar "$tar_args" "$package_filename" >/dev/null
 
-      if [ -z "$KEEP_BUILD_PATH" ]; then
-        rm -f "$package_filename"
-      else
-        true
-      fi
-    fi
-  } >&4 2>&1
+  if [ ! -d "$package_name" ]; then
+    extracted_dir="$(find_extracted_directory)"
+    mv "$extracted_dir" "$package_name"
+  fi
+
+  [ -n "$KEEP_BUILD_PATH" ] || rm -f "$package_filename"
 }
 
 find_extracted_directory() {
@@ -541,8 +532,8 @@ reuse_existing_tarball() {
   local cached_package_filename="${RUBY_BUILD_CACHE_PATH}/$package_filename"
 
   [ -e "$cached_package_filename" ] || return 1
-  verify_checksum "$cached_package_filename" "$checksum" >&4 2>&1 || return 1
-  ln -s "$cached_package_filename" "$package_filename" >&4 2>&1 || return 1
+  verify_checksum "$cached_package_filename" "$checksum" || return 1
+  ln -s "$cached_package_filename" "$package_filename" || return 1
 }
 
 download_tarball() {
@@ -552,10 +543,8 @@ download_tarball() {
   local package_filename="$2"
   local checksum="$3"
 
-  echo "-> $package_url" >&2
-
-  if http get "$package_url" "$package_filename" >&4 2>&1; then
-    verify_checksum "$package_filename" "$checksum" >&4 2>&1 || return 1
+  if http get "$package_url" "$package_filename"; then
+    verify_checksum "$package_filename" "$checksum" || return 1
   else
     echo "error: failed to download $package_filename" >&2
     return 1
@@ -563,9 +552,8 @@ download_tarball() {
 
   if [ -n "$RUBY_BUILD_CACHE_PATH" ]; then
     local cached_package_filename="${RUBY_BUILD_CACHE_PATH}/$package_filename"
-    { mv "$package_filename" "$cached_package_filename"
-      ln -s "$cached_package_filename" "$package_filename"
-    } >&4 2>&1 || return 1
+    mv "$package_filename" "$cached_package_filename"
+    ln -s "$cached_package_filename" "$package_filename"
   fi
 }
 
diff --git a/test/cache.bats b/test/cache.bats
index b99dbcdb..e43047c6 100644
--- a/test/cache.bats
+++ b/test/cache.bats
@@ -5,7 +5,7 @@ export RUBY_BUILD_SKIP_MIRROR=1
 export RUBY_BUILD_CACHE_PATH="$TMP/cache"
 
 @test "packages are saved to download cache" {
-  stub curl "-q -o * -*S* http://example.com/* : cp $FIXTURE_ROOT/\${5##*/} \$3"
+  stub curl "-q -fL -o * http://example.com/* : cp $FIXTURE_ROOT/\${5##*/} \$4"
 
   mkdir -p "$RUBY_BUILD_CACHE_PATH"
   install_fixture definitions/without-checksum
@@ -56,7 +56,7 @@ export RUBY_BUILD_CACHE_PATH="$TMP/cache"
 
   stub shasum true "echo invalid" "echo $checksum"
   stub curl "-*I* : true" \
-    "-q -o * -*S* https://?*/$checksum : cp $FIXTURE_ROOT/package-1.0.0.tar.gz \$3"
+    "-q -fL -o * https://?*/$checksum : cp $FIXTURE_ROOT/package-1.0.0.tar.gz \$4"
 
   mkdir -p "$RUBY_BUILD_CACHE_PATH"
   touch "${RUBY_BUILD_CACHE_PATH}/package-1.0.0.tar.gz"
@@ -74,7 +74,7 @@ export RUBY_BUILD_CACHE_PATH="$TMP/cache"
 
 
 @test "nonexistent cache directory is ignored" {
-  stub curl "-q -o * -*S* http://example.com/* : cp $FIXTURE_ROOT/\${5##*/} \$3"
+  stub curl "-q -fL -o * http://example.com/* : cp $FIXTURE_ROOT/\${5##*/} \$4"
 
   export RUBY_BUILD_CACHE_PATH="${TMP}/nonexistent"
 
diff --git a/test/checksum.bats b/test/checksum.bats
index d3130f6a..3c60ca1f 100644
--- a/test/checksum.bats
+++ b/test/checksum.bats
@@ -6,7 +6,7 @@ export RUBY_BUILD_CACHE_PATH=
 
 
 @test "package URL without checksum" {
-  stub curl "-q -o * -*S* http://example.com/* : cp $FIXTURE_ROOT/\${5##*/} \$3"
+  stub curl "-q -fL -o * http://example.com/* : cp $FIXTURE_ROOT/\${5##*/} \$4"
 
   install_fixture definitions/without-checksum
 
@@ -19,7 +19,7 @@ export RUBY_BUILD_CACHE_PATH=
 
 @test "package URL with valid checksum" {
   stub shasum true "echo ba988b1bb4250dee0b9dd3d4d722f9c64b2bacfc805d1b6eba7426bda72dd3c5"
-  stub curl "-q -o * -*S* http://example.com/* : cp $FIXTURE_ROOT/\${5##*/} \$3"
+  stub curl "-q -fL -o * http://example.com/* : cp $FIXTURE_ROOT/\${5##*/} \$4"
 
   install_fixture definitions/with-checksum
 
@@ -33,7 +33,7 @@ export RUBY_BUILD_CACHE_PATH=
 
 @test "package URL with invalid checksum" {
   stub shasum true "echo ba988b1bb4250dee0b9dd3d4d722f9c64b2bacfc805d1b6eba7426bda72dd3c5"
-  stub curl "-q -o * -*S* http://example.com/* : cp $FIXTURE_ROOT/\${5##*/} \$3"
+  stub curl "-q -fL -o * http://example.com/* : cp $FIXTURE_ROOT/\${5##*/} \$4"
 
   install_fixture definitions/with-invalid-checksum
 
@@ -47,7 +47,7 @@ export RUBY_BUILD_CACHE_PATH=
 
 @test "package URL with checksum but no shasum support" {
   stub shasum false
-  stub curl "-q -o * -*S* http://example.com/* : cp $FIXTURE_ROOT/\${5##*/} \$3"
+  stub curl "-q -fL -o * http://example.com/* : cp $FIXTURE_ROOT/\${5##*/} \$4"
 
   install_fixture definitions/with-checksum
 
@@ -61,7 +61,7 @@ export RUBY_BUILD_CACHE_PATH=
 
 @test "package URL with valid md5 checksum" {
   stub md5 true "echo 83e6d7725e20166024a1eb74cde80677"
-  stub curl "-q -o * -*S* http://example.com/* : cp $FIXTURE_ROOT/\${5##*/} \$3"
+  stub curl "-q -fL -o * http://example.com/* : cp $FIXTURE_ROOT/\${5##*/} \$4"
 
   install_fixture definitions/with-md5-checksum
 
@@ -75,7 +75,7 @@ export RUBY_BUILD_CACHE_PATH=
 
 @test "package URL with md5 checksum but no md5 support" {
   stub md5 false
-  stub curl "-q -o * -*S* http://example.com/* : cp $FIXTURE_ROOT/\${5##*/} \$3"
+  stub curl "-q -fL -o * http://example.com/* : cp $FIXTURE_ROOT/\${5##*/} \$4"
 
   install_fixture definitions/with-md5-checksum
 
@@ -89,7 +89,7 @@ export RUBY_BUILD_CACHE_PATH=
 
 @test "package with invalid checksum" {
   stub shasum true "echo invalid"
-  stub curl "-q -o * -*S* http://example.com/* : cp $FIXTURE_ROOT/\${5##*/} \$3"
+  stub curl "-q -fL -o * http://example.com/* : cp $FIXTURE_ROOT/\${5##*/} \$4"
 
   install_fixture definitions/with-checksum
 
@@ -125,7 +125,7 @@ DEF
   stub shasum true \
     "echo invalid" \
     "echo ba988b1bb4250dee0b9dd3d4d722f9c64b2bacfc805d1b6eba7426bda72dd3c5"
-  stub curl "-q -o * -*S* http://example.com/* : cp $FIXTURE_ROOT/\${5##*/} \$3"
+  stub curl "-q -fL -o * http://example.com/* : cp $FIXTURE_ROOT/\${5##*/} \$4"
 
   export -n RUBY_BUILD_CACHE_PATH
   export RUBY_BUILD_BUILD_PATH="${TMP}/build"
@@ -144,7 +144,7 @@ DEF
 }
 
 @test "package URL with checksum of unexpected length" {
-  stub curl "-q -o * -*S* http://example.com/* : cp $FIXTURE_ROOT/\${5##*/} \$3"
+  stub curl "-q -fL -o * http://example.com/* : cp $FIXTURE_ROOT/\${5##*/} \$4"
 
   run_inline_definition <<DEF
 install_package "package-1.0.0" "http://example.com/packages/package-1.0.0.tar.gz#checksum_of_unexpected_length" copy
diff --git a/test/fetch.bats b/test/fetch.bats
index 0ccf66c7..abc4909b 100644
--- a/test/fetch.bats
+++ b/test/fetch.bats
@@ -14,7 +14,6 @@ setup() {
 
   install_fixture definitions/without-checksum
   assert_failure
-  assert_output_contains "> http://example.com/packages/package-1.0.0.tar.gz"
   assert_output_contains "error: failed to download package-1.0.0.tar.gz"
 }
 
@@ -30,16 +29,11 @@ setup() {
 @test "using aria2c if available" {
   export RUBY_BUILD_ARIA2_OPTS=
   export -n RUBY_BUILD_HTTP_CLIENT
-  stub aria2c "--allow-overwrite=true --no-conf=true -o * http://example.com/* : cp $FIXTURE_ROOT/\${5##*/} \$4"
+  stub aria2c "--allow-overwrite=true --no-conf=true --console-log-level=warn --stderr -o * http://example.com/* : cp $FIXTURE_ROOT/\${7##*/} \$6"
 
   install_fixture definitions/without-checksum
   assert_success
-  assert_output <<OUT
-Downloading package-1.0.0.tar.gz...
--> http://example.com/packages/package-1.0.0.tar.gz
-Installing package-1.0.0...
-Installed package-1.0.0 to ${TMP}/install
-OUT
+  assert_output_contains "Downloading package-1.0.0.tar.gz..."
   unstub aria2c
 }
 
@@ -50,11 +44,7 @@ OUT
 install_git "package-dev" "http://example.com/packages/package.git" master copy
 DEF
   assert_success
-  assert_output <<OUT
-Cloning http://example.com/packages/package.git...
-Installing package-dev...
-Installed package-dev to ${TMP}/install
-OUT
+  assert_output_contains "Cloning http://example.com/packages/package.git..."
   unstub git
 }
 
@@ -68,10 +58,6 @@ OUT
 install_git "package-dev" "http://example.com/packages/package.git" master copy
 DEF
   assert_success
-  assert_output <<OUT
-Cloning http://example.com/packages/package.git...
-Installing package-dev...
-Installed package-dev to ${TMP}/install
-OUT
+  assert_output_contains "Cloning http://example.com/packages/package.git..."
   unstub git
 }
diff --git a/test/mirror.bats b/test/mirror.bats
index e1b8d2e1..6f83ceb6 100755
--- a/test/mirror.bats
+++ b/test/mirror.bats
@@ -8,10 +8,9 @@ export RUBY_BUILD_MIRROR_URL=http://mirror.example.com
 
 @test "package URL without checksum bypasses mirror" {
   stub shasum true
-  stub curl "-q -o * -*S* http://example.com/* : cp $FIXTURE_ROOT/\${5##*/} \$3"
+  stub curl "-q -fL -o * http://example.com/* : cp $FIXTURE_ROOT/\${5##*/} \$4"
 
   install_fixture definitions/without-checksum
-  echo "$output" >&2
 
   assert_success
   assert [ -x "${INSTALL_ROOT}/bin/package" ]
@@ -23,7 +22,7 @@ export RUBY_BUILD_MIRROR_URL=http://mirror.example.com
 
 @test "package URL with checksum but no shasum support bypasses mirror" {
   stub shasum false
-  stub curl "-q -o * -*S* http://example.com/* : cp $FIXTURE_ROOT/\${5##*/} \$3"
+  stub curl "-q -fL -o * http://example.com/* : cp $FIXTURE_ROOT/\${5##*/} \$4"
 
   install_fixture definitions/with-checksum
 
@@ -41,7 +40,7 @@ export RUBY_BUILD_MIRROR_URL=http://mirror.example.com
 
   stub shasum true "echo $checksum"
   stub curl "-*I* $mirror_url : true" \
-    "-q -o * -*S* $mirror_url : cp $FIXTURE_ROOT/package-1.0.0.tar.gz \$3"
+    "-q -fL -o * $mirror_url : cp $FIXTURE_ROOT/package-1.0.0.tar.gz \$4"
 
   install_fixture definitions/with-checksum
 
@@ -59,7 +58,7 @@ export RUBY_BUILD_MIRROR_URL=http://mirror.example.com
 
   stub shasum true "echo $checksum"
   stub curl "-*I* $mirror_url : false" \
-    "-q -o * -*S* http://example.com/* : cp $FIXTURE_ROOT/\${5##*/} \$3"
+    "-q -fL -o * http://example.com/* : cp $FIXTURE_ROOT/\${5##*/} \$4"
 
   install_fixture definitions/with-checksum
 
@@ -77,11 +76,10 @@ export RUBY_BUILD_MIRROR_URL=http://mirror.example.com
 
   stub shasum true "echo invalid" "echo $checksum"
   stub curl "-*I* $mirror_url : true" \
-    "-q -o * -*S* $mirror_url : cp $FIXTURE_ROOT/package-1.0.0.tar.gz \$3" \
-    "-q -o * -*S* http://example.com/* : cp $FIXTURE_ROOT/\${5##*/} \$3"
+    "-q -fL -o * $mirror_url : cp $FIXTURE_ROOT/package-1.0.0.tar.gz \$4" \
+    "-q -fL -o * http://example.com/* : cp $FIXTURE_ROOT/\${5##*/} \$4"
 
   install_fixture definitions/with-checksum
-  echo "$output" >&2
 
   assert_success
   assert [ -x "${INSTALL_ROOT}/bin/package" ]
@@ -97,7 +95,7 @@ export RUBY_BUILD_MIRROR_URL=http://mirror.example.com
 
   stub shasum true "echo $checksum"
   stub curl "-*I* : true" \
-    "-q -o * -*S* https://?*/$checksum : cp $FIXTURE_ROOT/package-1.0.0.tar.gz \$3" \
+    "-q -fL -o * https://?*/$checksum : cp $FIXTURE_ROOT/package-1.0.0.tar.gz \$4" \
 
   install_fixture definitions/with-checksum
 
@@ -114,7 +112,7 @@ export RUBY_BUILD_MIRROR_URL=http://mirror.example.com
   local checksum="ba988b1bb4250dee0b9dd3d4d722f9c64b2bacfc805d1b6eba7426bda72dd3c5"
 
   stub shasum true "echo $checksum"
-  stub curl "-q -o * -*S* https://cache.ruby-lang.org/* : cp $FIXTURE_ROOT/\${5##*/} \$3"
+  stub curl "-q -fL -o * https://cache.ruby-lang.org/* : cp $FIXTURE_ROOT/\${5##*/} \$4"
 
   run_inline_definition <<DEF
 install_package "package-1.0.0" "https://cache.ruby-lang.org/packages/package-1.0.0.tar.gz#ba988b1bb4250dee0b9dd3d4d722f9c64b2bacfc805d1b6eba7426bda72dd3c5" copy
@@ -135,7 +133,7 @@ DEF
 
   stub shasum true "echo $checksum"
   stub curl "-*I* $RUBY_BUILD_MIRROR_PACKAGE_URL : true" \
-    "-q -o * -*S* $RUBY_BUILD_MIRROR_PACKAGE_URL : cp $FIXTURE_ROOT/package-1.0.0.tar.gz \$3"
+    "-q -fL -o * $RUBY_BUILD_MIRROR_PACKAGE_URL : cp $FIXTURE_ROOT/package-1.0.0.tar.gz \$4"
 
   install_fixture definitions/with-checksum
 
@@ -154,7 +152,7 @@ DEF
 
   stub shasum true "echo $checksum"
   stub curl "-*I* $RUBY_BUILD_MIRROR_PACKAGE_URL : false" \
-    "-q -o * -*S* http://example.com/* : cp $FIXTURE_ROOT/\${5##*/} \$3"
+    "-q -fL -o * http://example.com/* : cp $FIXTURE_ROOT/\${5##*/} \$4"
 
   install_fixture definitions/with-checksum
 
@@ -173,11 +171,10 @@ DEF
 
   stub shasum true "echo invalid" "echo $checksum"
   stub curl "-*I* $RUBY_BUILD_MIRROR_PACKAGE_URL : true" \
-    "-q -o * -*S* $RUBY_BUILD_MIRROR_PACKAGE_URL : cp $FIXTURE_ROOT/package-1.0.0.tar.gz \$3" \
-    "-q -o * -*S* http://example.com/* : cp $FIXTURE_ROOT/\${5##*/} \$3"
+    "-q -fL -o * $RUBY_BUILD_MIRROR_PACKAGE_URL : cp $FIXTURE_ROOT/package-1.0.0.tar.gz \$4" \
+    "-q -fL -o * http://example.com/* : cp $FIXTURE_ROOT/\${5##*/} \$4"
 
   install_fixture definitions/with-checksum
-  echo "$output" >&2
 
   assert_success
   assert [ -x "${INSTALL_ROOT}/bin/package" ]