Keep mirror up-to-date using Travis CI

When new Ruby definitions get added, our current CloudFront mirror may
get out of date. This adds a task to the CI process that detects
added/changed package URLs and uploads them to our S3 bucket.

Fixes #392, references #395

.travis.yml

@@ -1,4 +1,8 @@
 install: git clone https://github.com/sstephenson/bats.git
-script: bats/bin/bats -t test
+script: script/test
-# skips unnecessary Ruby-specific setup
 language: c
+env:
+  global:
+    - AMAZON_S3_BUCKET=ruby-build-mirror
+    - AMAZON_ACCESS_KEY_ID=AKIAJKAUQVHU6X4CODDQ
+    - secure: LTSvDP2o72nbECDwWsfwnsiETF4VpqrYN3y/ve68AZIMzfNWDB5vhqzMLU1ltFnSNxd71gTCGX2OEcsxdrfnG+Msu52v8FtJ7lz/b9xn83gGYrGnmEMzARtd1fnuzlWQh/1eNL9jrNl8FDhgjoTqKl2gF6fZBsQxcHRnvRSXcqE=

script/mirror

@@ -0,0 +1,86 @@
+#!/usr/bin/env bash
+# Usage: script/mirror update <COMMIT-RANGE>
+#        script/mirror stats
+set -e
+
+commit_range="${1?}"
+
+eval "$(grep RUBY_BUILD_MIRROR_URL= ./bin/ruby-build | head -1)"
+
+test_mirrored() {
+  curl -qsSfIL "$RUBY_BUILD_MIRROR_URL/$1" >/dev/null 2>&1
+}
+
+compute_md5() {
+  local output="$(openssl md5)"
+  echo "${output##* }" | tr '[A-Z]' '[a-z]'
+}
+
+download_package() {
+  curl -qsSfL -o "$2" "$1"
+}
+
+download_and_verify() {
+  local checksum
+  local url="$1"
+  local file="$2"
+  local expected="$3"
+  download_package "$url" "$file"
+  checksum="$(compute_md5 < "$file")"
+  if [ "$checksum" != "$expected" ]; then
+    echo "Error: $url doesn't match its checksum $expected" >&2
+    return 1
+  fi
+}
+
+changed_files() {
+  git diff --name-only --diff-filter=ACMR "$@"
+}
+
+potentially_new_packages() {
+  extract_urls $(changed_files "$1" -- ./share/ruby-build)
+}
+
+extract_urls() {
+  grep -hoe 'http[^"]\+#[^"]\+' "$@"
+}
+
+update() {
+  local url
+  local checksum
+  local file
+  for url in $(potentially_new_packages "$1"); do
+    checksum="${url#*#}"
+    url="${url%#*}"
+    if test_mirrored "$checksum"; then
+      echo "Already mirrored: $url"
+    else
+      echo "Mirroring: $url"
+      file="${TMPDIR:-/tmp}/$checksum"
+      download_and_verify "$url" "$file" "$checksum"
+      ./script/s3-put "$file" "${AMAZON_S3_BUCKET?}"
+    fi
+  done
+}
+
+stats() {
+  local packages=( $(extract_urls ./share/ruby-build/*) )
+  local total="${#packages[@]}"
+  local confirmed=0
+  local checksum
+  for url in "${packages[@]}"; do
+    checksum="${url#*#}"
+    if test_mirrored "$checksum"; then
+      confirmed="$((confirmed + 1))"
+    else
+      echo "failed: $url" >&2
+    fi
+    echo -n "."
+  done
+  echo
+  echo "$confirmed/$total mirrored"
+}
+
+cmd="${1?}"
+shift 1
+"$cmd" "$@"

script/s3-put

@@ -0,0 +1,63 @@
+#!/usr/bin/env bash
+# Usage: s3-put <FILE> <S3_BUCKET> [<CONTENT_TYPE>]
+#
+# Uploads a file to the Amazon S3 service.
+#
+# Depends on AWS credentials being set via env:
+# - AMAZON_ACCESS_KEY_ID
+# - AMAZON_SECRET_ACCESS_KEY
+#
+# Outputs the URL of the newly uploaded file.
+set -e
+
+authorization() {
+  local signature="$(string_to_sign | hmac_sha1 | base64)"
+  echo "AWS ${AMAZON_ACCESS_KEY_ID?}:${signature}"
+}
+
+hmac_sha1() {
+  openssl dgst -binary -sha1 -hmac "${AMAZON_SECRET_ACCESS_KEY?}"
+}
+
+base64() {
+  openssl enc -base64
+}
+
+bin_md5() {
+  openssl dgst -binary -md5
+}
+
+string_to_sign() {
+  echo "$http_method"
+  echo "$content_md5"
+  echo "$content_type"
+  echo "$date"
+  echo "x-amz-acl:$acl"
+  printf "/$bucket/$remote_path"
+}
+
+date_string() {
+  LC_TIME=C date "+%a, %d %h %Y %T %z"
+}
+
+file="$1"
+bucket="$2"
+content_type="$3"
+
+http_method=PUT
+acl="public-read"
+remote_path="${file##*/}"
+content_md5="$(bin_md5 < "$file" | base64)"
+date="$(date_string)"
+
+url="https://$bucket.s3.amazonaws.com/$remote_path"
+
+curl -qsSf -T "$file" \
+  -H "Authorization: $(authorization)" \
+  -H "x-amz-acl: $acl" \
+  -H "Date: $date" \
+  -H "Content-MD5: $content_md5" \
+  -H "Content-Type: $content_type" \
+  "$url"
+
+echo "$url"

script/test

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+set -e
+
+[ -d ./bats/bin ] && export PATH=./bats/bin:"$PATH"
+
+bats -t test
+
+if [ "$TRAVIS_SECURE_ENV_VARS" = "true" ]; then
+  ./script/mirror update "$TRAVIS_COMMIT_RANGE"
+fi