Google Cloud Storage origin (private)

Use AWS compat mode to make authenticated requests to your GCS bucket.

VCL

Use this solution in your VCL service (click RUN below to test this solution or clone it to make changes):

Compute@Edge

Use this solution in your Compute@Edge service:

  1. Rust
  2. JavaScript
Cargo.toml
Rust
[dependencies]
anyhow = "1.0.28"
chrono = "0.4"
fastly = "0.7.0"
hex = "0.4"
hmac-sha256 = "0.1"
log-fastly = "0.2"
log = "0.4"
urlencoding = "1.1"
main.rs
Rust
/// GCP Backend Name
const BACKEND_GCS: &str = "storage.googleapis.com";
// Hash of empty string, used for authentication string generation
// caculated from hex::encode(Hash::hash("".as_bytes()));
const EMPTY_HASH: &str = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855";
/// Google Cloud Storage Configuration for Interoperability API
struct GcsConfig {
access_key_id: String,
secret_access_key: String,
region: String,
bucket: String,
}
impl GcsConfig {
/// Load the GCS configuration.
///
/// This assumes an Edge Dictionary named "gcs_config" is attached to this service,
/// with entries for access_key_id, secret_key, region and bucket.
fn load_config() -> Self {
let dict = Dictionary::open("gcs_config");
Self {
access_key_id: dict.get("access_key_id").expect("access key id configured"),
secret_access_key: dict
.get("secret_access_key")
.expect("secret access key configured"),
region: dict.get("region").expect("region configured"),
bucket: dict.get("bucket").expect("bucket configured"),
}
}
}
#[fastly::main]
fn main(mut req: Request) -> Result<Response, Error> {
log_fastly::init_simple("my_log", log::LevelFilter::Info);
fastly::log::set_panic_endpoint("my_log")?;
let gcs_config = GcsConfig::load_config();
// Only generate authorize header for GET and HEAD request
// Pass PURGE to backend as it is
// Block other kinds of method
let resp = match req.get_method() {
&Method::GET | &Method::HEAD => {
req = authorize_gcs_request(
req,
&gcs_config.access_key_id,
&gcs_config.secret_access_key,
&gcs_config.region,
&gcs_config.bucket,
)?;
req.send(BACKEND_GCS)?
}
m if m == "PURGE" => {
// When doing the purge, we need to make sure the cache key matches
// Cache key is cacualted from URL and HOST header of request sent to backend
// URL of backend request has no query string
req.remove_query();
// HOST header of backend request is like {bucket}.storage.googleapis.com
req.set_header(
header::HOST,
format!("{}.storage.googleapis.com", gcs_config.bucket),
);
req.send(BACKEND_GCS)?
}
_ => Response::from_status(StatusCode::METHOD_NOT_ALLOWED)
.with_header(header::ALLOW, "GET, HEAD"),
};
Ok(resp)
}
/// Generate HMAC hash of message with key
fn sign(key: &[u8], message: &str) -> [u8; 32] {
HMAC::mac(message.as_bytes(), key)
}
/// Generate Authorization headers of the S3 request
fn authorize_gcs_request(
mut req: Request,
access_key_id: &str,
secret_access_key: &str,
region: &str,
bucket: &str,
) -> Result<Request> {
// Ignore the query string from client
req.remove_query();
let canonical_querystring = "";
let x_amz_content_sha256 = EMPTY_HASH;
let amz_date = chrono::Utc::now().format("%Y%m%dT%H%M%SZ").to_string();
// Prepare authentication related information
let host = format!("{}.storage.googleapis.com", bucket);
// No need to URL encode twice for S3 object
// https://docs.aws.amazon.com/general/latest/gr/sigv4-create-canonical-request.html
// "Each path segment must be URI-encoded twice (EXCEPT for Amazon S3 which only gets URI-encoded once)"
// Do url decode and re-encode in case some client are not do url encoding according S3 spec
// https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-header-based-auth.html
let path_url_decoded = urlencoding::decode(req.get_path())?;
let path_url_encoded = urlencoding::encode(&path_url_decoded);
// No url encode of "/" in object id
let canonical_uri = path_url_encoded.replace("%2F", "/");
// Generate Authorization header value
// Force the method to GET
let authorization_value = generate_s3_authorization_header(
access_key_id,
secret_access_key,
region,
&host,
&canonical_uri,
canonical_querystring,
x_amz_content_sha256,
"GET",
&amz_date,
"storage",
);
req.set_method(Method::GET);
// Add authorization related headers to request
req.set_header(header::HOST, &host);
req.set_header(header::AUTHORIZATION, &authorization_value);
req.set_header("x-amz-content-sha256", x_amz_content_sha256);
req.set_header("x-amz-date", &amz_date);
log::info!(
"Path: {}, Host: {}, x-amz-date: {}, Authorization: {}",
req.get_path(),
host,
amz_date,
authorization_value,
);
Ok(req)
}
#[allow(clippy::too_many_arguments)]
/// Generate authoriation header for S3 object access
/// https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-header-based-auth.html
fn generate_s3_authorization_header(
access_key_id: &str,
secret_access_key: &str,
region: &str,
host: &str,
canonical_uri: &str,
canonical_querystring: &str,
x_amz_content_sha256: &str,
method: &str,
amz_date: &str,
service: &str,
) -> String {
let canonical_headers = format!(
"host:{}\nx-amz-content-sha256:{}\nx-amz-date:{}\n",
host, x_amz_content_sha256, amz_date
);
let signed_headers = "host;x-amz-content-sha256;x-amz-date";
let canonical_request = format!(
"{}\n{}\n{}\n{}\n{}\n{}",
method,
canonical_uri,
canonical_querystring,
canonical_headers,
signed_headers,
x_amz_content_sha256
);
// credential_date format is YYYYMMDD
let credential_date = &amz_date[..8];
let credential_scope = format!("{}/{}/{}/aws4_request", credential_date, region, service);
// Compose string to be signed
let canonical_request_hash = hex::encode(Hash::hash(canonical_request.as_bytes()));
let string_to_sign = format!(
"AWS4-HMAC-SHA256\n{}\n{}\n{}",
amz_date, credential_scope, canonical_request_hash
);
// Generate signing key
let aws4_secret = format!("AWS4{}", secret_access_key).into_bytes();
let date_key = sign(&aws4_secret, credential_date);
let region_key = sign(&date_key, region);
let service_key = sign(&region_key, service);
let signing_key = sign(&service_key, "aws4_request");
// Generate signature
let signature = hex::encode(sign(&signing_key, &string_to_sign));
// Compose authorization header value
format!(
"AWS4-HMAC-SHA256 Credential={}/{},SignedHeaders={},Signature={}",
access_key_id, credential_scope, signed_headers, signature
)
}
This page is part of a series in the Static content topic.