V4SignedURL

v4_signed_urls.rb

require "base64"
require "cgi"
require "json"
require "net/http"
require "openssl"
require "uri"
require "time"

def service_account_signer private_key
    # Create a signer
    signer = OpenSSL::PKey::RSA.new private_key

    # Sign string to sign
    lambda do |string_to_sign|
        signature = signer.sign(OpenSSL::Digest::SHA256.new, string_to_sign).unpack('H*').first
    end
end

def iam_signer project_id, client_email, access_token
    lambda do |string_to_sign|
        begin
            uri = URI("https://iam.googleapis.com/v1/projects/-/serviceAccounts/#{client_email}:signBlob")
            http = Net::HTTP.new(uri.host, uri.port)
            http.use_ssl = true
            #http.set_debug_output($stdout)
            header = {}
            header["Authorization"] = "Bearer #{access_token}"
            header["Content-type"] = "application/json"
            req = Net::HTTP::Post.new(uri, header)
            req.body = {"bytesToSign" => Base64.strict_encode64(string_to_sign)}.to_json
            res = http.request req
            unless res.kind_of? Net::HTTPSuccess
                fail "Unable to sign string #{res.body}"
            end
            puts res.body
            Base64.strict_decode64(JSON.parse(res.body)["signature"]).unpack('H*').first
        rescue => err
            fail "Error occurred: #{err}"
        end
    end
end

def create_signed_url request_method, bucket_name, object_name,
                      expiration, query_parameters:nil, headers:nil,
                      client_email: nil, access_token: nil, project_id: nil, private_key: nil,
                      service_account_file: nil, region_name: "auto"
    # Verify at least one is provided.
    if access_token.nil? || client_email.nil? || project_id.nil?
        if client_email.nil? || private_key.nil?
                if service_account_file.nil?
                fail "Either an Access Token and Client Email, or a Google Service Account is required."
            end
        end
    end

    # Select appropriate signer
    signer = nil
    if !access_token.nil? && !client_email.nil? && !project_id.nil?
        # Use access token
        signer = iam_signer project_id, CGI.escape(client_email), access_token
    elsif !service_account_file.nil?
        # Use service account
        # Parse the Service Account and get client id and private key
        google_service_account = JSON.parse(File.read(service_account_file))
        client_email = google_service_account["client_email"]
        private_key = google_service_account["private_key"]
        signer = service_account_signer private_key
    end
    # else client_email and private_key are provided as provided.

    if expiration > 604800
        fail "Expiration time can't be longer than a week"
    end

    # Need to diff the untracked parameters in the request...
    host = "storage.googleapis.com"
    path = "#{bucket_name}/#{object_name}"
    datetime_now = Time.now.utc
    goog_date = datetime_now.strftime("%Y%m%dT%H%M%SZ")
    datestamp = datetime_now.strftime("%Y%m%d")
    algorithm = "GOOG4-RSA-SHA256"
    # goog4_request is not checked.
    credential_scope = "#{datestamp}/#{region_name}/storage/goog4_request"
    # Headers needs to be in alpha order.
    canonical_headers = headers || {}
    canonical_headers["host"] = host

    canonical_headers = canonical_headers.sort_by { |k, v| k.downcase }.to_h
    canonical_headers_str = ""
    canonical_headers.each { |k, v| canonical_headers_str += "#{k}:#{v}\n" }
    signed_headers_str = ""
    canonical_headers.each { |k, v| signed_headers_str += "#{k};" }
    signed_headers_str = signed_headers_str.chomp(';') # remove trailing ';'

    # Begin constructing string_to_sign
    # Needs to be in alpha order
    credential = CGI.escape(client_email + "/" + credential_scope)
    query_parameters ||={}
    query_parameters["X-Goog-Algorithm"] = algorithm
    query_parameters["X-Goog-Credential"] = credential
    query_parameters["X-Goog-Date"] = goog_date
    query_parameters["X-Goog-Expires"] = expiration
    query_parameters["X-Goog-SignedHeaders"] = CGI.escape(signed_headers_str)
    query_parameters = query_parameters.sort_by { |k, v| k.downcase }.to_h
    canonical_querystring = ""
    query_parameters.each { |k, v| canonical_querystring += "#{k}=#{v}&" }
    canonical_querystring = canonical_querystring.chomp("&") # remove trailing '&'

    # From AWS: You don't include a payload hash in the Canonical Request,
    # because when you create a presigned URL, you don't know the payload
    # content because the URL is used to upload an arbitrary payload. Instead,
    # you use a constant string UNSIGNED-PAYLOAD.
    canonical_request = [request_method,
                         path,
                         canonical_querystring,
                         canonical_headers_str,
                         signed_headers_str,
                         "UNSIGNED-PAYLOAD"].join("\n")

    # Construct string to sign
    string_to_sign = [algorithm,
                      goog_date,
                      credential_scope,
                      Digest::SHA256.hexdigest(canonical_request)].join("\n")
    #puts string_to_sign
    # Sign string
    signature = signer.call(string_to_sign)

    # Construct signed URL
    "https://#{host}#{path}?#{canonical_querystring}&X-Goog-Signature=#{signature}"
end


### Used for testing
def test_signed_url_upload signed_url, file, headers
    # Ruby example of getting the resumable session.
    # Create the HTTP object
    uri = URI.parse(signed_url)
    https = Net::HTTP.new(uri.host, uri.port)
    https.use_ssl = true
    # https.set_debug_output $stderr # debugging purposes only

    # Prepare POST request to get resumable session
    headers ||= {}
    headers["x-goog-resumable"] = "start"

    request = Net::HTTP::Post.new(uri.request_uri, headers)
    request.content_type = "text/plain"

    # Send POST request to get resumable session URL.
    response = https.request(request)
    unless response.kind_of? Net::HTTPSuccess
        fail response.body
    end
    # Process response
    resumable_session_url = response["location"]
    # chunk_size can be increased but uploaded chunks need to be a multiple of 256 kibibytes or 262,144 bytes.
    chunk_size = 256*1024
    last_byte = 0
    total_bytes = 0
    headers = {}

    while !file.eof?
        chunk = file.read(chunk_size)
        total_bytes += chunk.size

        if chunk.size == chunk_size && !file.eof?
            headers["Content-Range"] = "bytes #{last_byte}-#{last_byte+chunk_size-1}/*"
        else
            headers["Content-Range"] = "bytes #{last_byte}-#{last_byte+chunk.size-1}/#{total_bytes}"
        end
        headers["Content-Length"] = chunk.size.to_s
        last_byte += chunk.size

        request = Net::HTTP::Put.new(resumable_session_url, headers)
        request.content_type = "text/plain"
        request.body = chunk
        response = https.request(request)

        case response.code
        when "200"
            puts "Completed upload"
        when "308"
            puts response["Range"]
        else
            raise "Unhandled response code #{response.code}"
        end
    end
end

if $PROGRAM_NAME == __FILE__
    #### Using a Service Account (POST)
    service_account = "spec-test-ruby-samples-3fb0d39a74ab.json"
    request_method = "POST"
    bucket_name = "jessefrank2"
    object_name = "test_lifecycle.json"
    expiration = 60*60*24
    query_parameters = {'x-goog-hello' => 'hello'}
    headers = {'x-goog-hello' => 'hello'}

    post_signed_url = create_signed_url request_method, bucket_name, object_name, expiration,
        service_account_file: service_account,
        query_parameters: query_parameters,
        headers: headers
    ### Test signed POST request using a resumable upload
    File.open(__FILE__) do |f|
        test_signed_url_upload(post_signed_url, f, headers)
    end

    #### Using the IAM service to sign blob (GET)
    # 1. Activate service account "gcloud auth activate-service-account --key-file=service-account.json"
    # 2. Get Access Token "gcloud auth print-access-token"
    # This should be retrieved using GCE Metadata server..
    # https://cloud.google.com/compute/docs/access/create-enable-service-accounts-for-instances#applications
    access_token = "ya29...."
    client_email = "test-account@spec-test-ruby-samples.iam.gserviceaccount.com"
    project_id = "spec-test-ruby-samples"
    request_method = "GET"
    bucket_name = "jessefrank2"
    object_name = "test_lifecycle.json"
    expiration = 60*60*24
    region_name = "auto" # Only necessary for creating a signed URL to create a bucket.
    query_parameters = nil
    headers = nil

    puts create_signed_url request_method, bucket_name, object_name, expiration,
        client_email: client_email,
        access_token: access_token,
        project_id: project_id,
        query_parameters: query_parameters,
        headers: headers,
        region_name: region_name
end