diff --git a/.travis.yml b/.travis.yml index 04587654..2e8217e7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,4 +1,8 @@ install: git clone https://github.com/sstephenson/bats.git -script: bats/bin/bats -t test -# skips unnecessary Ruby-specific setup +script: script/test language: c +env: + global: + - AMAZON_S3_BUCKET=ruby-build-mirror + - AMAZON_ACCESS_KEY_ID=AKIAJKAUQVHU6X4CODDQ + - secure: LTSvDP2o72nbECDwWsfwnsiETF4VpqrYN3y/ve68AZIMzfNWDB5vhqzMLU1ltFnSNxd71gTCGX2OEcsxdrfnG+Msu52v8FtJ7lz/b9xn83gGYrGnmEMzARtd1fnuzlWQh/1eNL9jrNl8FDhgjoTqKl2gF6fZBsQxcHRnvRSXcqE= diff --git a/script/mirror b/script/mirror new file mode 100755 index 00000000..a2bc0c11 --- /dev/null +++ b/script/mirror @@ -0,0 +1,86 @@ +#!/usr/bin/env bash +# Usage: script/mirror update +# script/mirror stats +set -e + +commit_range="${1?}" + +eval "$(grep RUBY_BUILD_MIRROR_URL= ./bin/ruby-build | head -1)" + +test_mirrored() { + curl -qsSfIL "$RUBY_BUILD_MIRROR_URL/$1" >/dev/null 2>&1 +} + +compute_md5() { + local output="$(openssl md5)" + echo "${output##* }" | tr '[A-Z]' '[a-z]' +} + +download_package() { + curl -qsSfL -o "$2" "$1" +} + +download_and_verify() { + local checksum + local url="$1" + local file="$2" + local expected="$3" + download_package "$url" "$file" + checksum="$(compute_md5 < "$file")" + if [ "$checksum" != "$expected" ]; then + echo "Error: $url doesn't match its checksum $expected" >&2 + return 1 + fi +} + +changed_files() { + git diff --name-only --diff-filter=ACMR "$@" +} + +potentially_new_packages() { + extract_urls $(changed_files "$1" -- ./share/ruby-build) +} + +extract_urls() { + grep -hoe 'http[^"]\+#[^"]\+' "$@" +} + +update() { + local url + local checksum + local file + for url in $(potentially_new_packages "$1"); do + checksum="${url#*#}" + url="${url%#*}" + if test_mirrored "$checksum"; then + echo "Already mirrored: $url" + else + echo "Mirroring: $url" + file="${TMPDIR:-/tmp}/$checksum" + download_and_verify "$url" "$file" "$checksum" + ./script/s3-put "$file" "${AMAZON_S3_BUCKET?}" + fi + done +} + +stats() { + local packages=( $(extract_urls ./share/ruby-build/*) ) + local total="${#packages[@]}" + local confirmed=0 + local checksum + for url in "${packages[@]}"; do + checksum="${url#*#}" + if test_mirrored "$checksum"; then + confirmed="$((confirmed + 1))" + else + echo "failed: $url" >&2 + fi + echo -n "." + done + echo + echo "$confirmed/$total mirrored" +} + +cmd="${1?}" +shift 1 +"$cmd" "$@" diff --git a/script/s3-put b/script/s3-put new file mode 100755 index 00000000..5efdfce7 --- /dev/null +++ b/script/s3-put @@ -0,0 +1,63 @@ +#!/usr/bin/env bash +# Usage: s3-put [] +# +# Uploads a file to the Amazon S3 service. +# +# Depends on AWS credentials being set via env: +# - AMAZON_ACCESS_KEY_ID +# - AMAZON_SECRET_ACCESS_KEY +# +# Outputs the URL of the newly uploaded file. +set -e + +authorization() { + local signature="$(string_to_sign | hmac_sha1 | base64)" + echo "AWS ${AMAZON_ACCESS_KEY_ID?}:${signature}" +} + +hmac_sha1() { + openssl dgst -binary -sha1 -hmac "${AMAZON_SECRET_ACCESS_KEY?}" +} + +base64() { + openssl enc -base64 +} + +bin_md5() { + openssl dgst -binary -md5 +} + +string_to_sign() { + echo "$http_method" + echo "$content_md5" + echo "$content_type" + echo "$date" + echo "x-amz-acl:$acl" + printf "/$bucket/$remote_path" +} + +date_string() { + LC_TIME=C date "+%a, %d %h %Y %T %z" +} + +file="$1" +bucket="$2" +content_type="$3" + +http_method=PUT +acl="public-read" +remote_path="${file##*/}" +content_md5="$(bin_md5 < "$file" | base64)" +date="$(date_string)" + +url="https://$bucket.s3.amazonaws.com/$remote_path" + +curl -qsSf -T "$file" \ + -H "Authorization: $(authorization)" \ + -H "x-amz-acl: $acl" \ + -H "Date: $date" \ + -H "Content-MD5: $content_md5" \ + -H "Content-Type: $content_type" \ + "$url" + +echo "$url" diff --git a/script/test b/script/test new file mode 100755 index 00000000..6e708f1e --- /dev/null +++ b/script/test @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +set -e + +[ -d ./bats/bin ] && export PATH=./bats/bin:"$PATH" + +bats -t test + +if [ "$TRAVIS_SECURE_ENV_VARS" = "true" ]; then + ./script/mirror update "$TRAVIS_COMMIT_RANGE" +fi