diff --git a/Cargo.lock b/Cargo.lock index c0e896fa10..7cf295fcb5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -25,7 +25,7 @@ checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" dependencies = [ "cfg-if", "cipher", - "cpufeatures", + "cpufeatures 0.2.17", ] [[package]] @@ -245,6 +245,12 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "base64" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5" + [[package]] name = "base64-simd" version = "0.8.0" @@ -396,6 +402,17 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" +[[package]] +name = "chacha20" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65c35e4b699c7e15ccbe7ee35c005e4fc0a278d22238a2857e6ce2dadeda1b06" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.1", + "rand_core 0.10.1", +] + [[package]] name = "chrono" version = "0.4.45" @@ -513,8 +530,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ce2548391e9c1929c21bf6aa2680af86fe4c1b33e6cea9ac1cfeec0bd11218cf" dependencies = [ "compression-core", + "flate2", + "memchr", "zstd", - "zstd-safe", + "zstd-safe 7.4.0", ] [[package]] @@ -604,6 +623,15 @@ dependencies = [ "libc", ] +[[package]] +name = "cpufeatures" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ca28b0ae3115b884660db4118d803791fd6756b6e88f39c0f3f7859060d7566" +dependencies = [ + "libc", +] + [[package]] name = "crc" version = "3.4.0" @@ -733,7 +761,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "97fb8b7c4503de7d6ae7b42ab72a5a59857b4c937ec27a3d4539dba95b5ab2be" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.2.17", "curve25519-dalek-derive", "digest", "fiat-crypto", @@ -1263,6 +1291,7 @@ dependencies = [ "cfg-if", "libc", "r-efi 6.0.0", + "rand_core 0.10.1", ] [[package]] @@ -1785,9 +1814,9 @@ dependencies = [ [[package]] name = "infer" -version = "0.19.0" +version = "0.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a588916bfdfd92e71cacef98a63d9b1f0d74d6599980d11894290e7ddefffcf7" +checksum = "f4200d433cbd5178df7797c9c2e75b348b728e39631cf14520d1e2fc424201f4" [[package]] name = "inout" @@ -1900,9 +1929,9 @@ dependencies = [ [[package]] name = "jsonwebtoken" -version = "10.4.0" +version = "11.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eba32bfb4ffdeaca3e34431072faf01745c9b26d25504aa7a6cf5684334fc4fc" +checksum = "e75fe14a82d81e5f5af639997db37d8b96045938a7ac6ab18cdbe1c7467e05e1" dependencies = [ "base64 0.22.1", "ed25519-dalek", @@ -2078,9 +2107,9 @@ checksum = "4facc753ae494aeb6e3c22f839b158aebd4f9270f55cd3c79906c45476c47ab4" [[package]] name = "mediatype" -version = "0.21.0" +version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "120fa187be19d9962f0926633453784691731018a2bf936ddb4e29101b79c4a7" +checksum = "0695a693e38d35838dd34bc3ab19911395f6837ccf55c7271189cfb12cb8b57f" [[package]] name = "memchr" @@ -2484,9 +2513,9 @@ dependencies = [ [[package]] name = "objectstore-client" -version = "0.1.12" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2713028097b2f0f00696d7d389a877f7370c02813548124c4422941f669732f3" +checksum = "d73524fb13307fc0b5f9de80c280578951bc5e6a05e5681fba5824ca73fe140a" dependencies = [ "async-compression", "bytes", @@ -2495,27 +2524,31 @@ dependencies = [ "jsonwebtoken", "multer", "objectstore-types", - "percent-encoding", + "rand 0.10.4", "reqwest", "sentry-core 0.48.3", "serde", + "serde_json", + "tempfile", "thiserror 2.0.18", "tokio", "tokio-util", "url", - "zstd-safe", + "zstd-safe 8.3.0", ] [[package]] name = "objectstore-types" -version = "0.1.12" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c83aedfcc050322caefbfdba0372518f316624995f04f97a1f00cafb08ce1d0c" +checksum = "574c442bb1b29763ba22ac21599616a505bb8a318fcb4bf1dc5b1f344bb3b540" dependencies = [ + "base64 0.23.1", + "ed25519-dalek", "http", "humantime", - "humantime-serde", "mediatype", + "percent-encoding", "serde", "thiserror 2.0.18", ] @@ -3067,6 +3100,17 @@ dependencies = [ "rand_core 0.9.5", ] +[[package]] +name = "rand" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f6fa277c67e6ea590466f04c3fe82cb509f0864fe470643e5570de9514a9b11" +dependencies = [ + "chacha20", + "getrandom 0.4.3", + "rand_core 0.10.1", +] + [[package]] name = "rand_chacha" version = "0.3.1" @@ -3105,6 +3149,12 @@ dependencies = [ "getrandom 0.3.4", ] +[[package]] +name = "rand_core" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" + [[package]] name = "range-collections" version = "0.2.4" @@ -3229,6 +3279,7 @@ dependencies = [ "rustls-pki-types", "serde", "serde_json", + "serde_urlencoded", "sync_wrapper", "tokio", "tokio-native-tls", @@ -3845,7 +3896,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.2.17", "digest", ] @@ -3865,7 +3916,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.2.17", "digest", ] @@ -4216,7 +4267,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom 0.3.4", + "getrandom 0.4.3", "once_cell", "rustix 1.1.4", "windows-sys 0.61.2", @@ -4373,7 +4424,6 @@ dependencies = [ "bytes", "futures-core", "futures-sink", - "futures-util", "pin-project-lite", "tokio", ] @@ -5349,7 +5399,7 @@ version = "0.13.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cdb02c3132d251ab1606e9fc913a7da6c6ed08cc263df15edf082e1dc497d157" dependencies = [ - "zstd-safe", + "zstd-safe 7.4.0", ] [[package]] @@ -5361,11 +5411,20 @@ dependencies = [ "zstd-sys", ] +[[package]] +name = "zstd-safe" +version = "8.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a9ea9cc2576c2fcc104e9cec04f46e3ede564f4fbc206831e4ef7e859a4140d" +dependencies = [ + "zstd-sys", +] + [[package]] name = "zstd-sys" -version = "2.1.2+zstd.1.5.7" +version = "2.1.3+zstd.1.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "481ada9c543b2323cbb7bbdbb5249177fbf0188ed959ed2c15e4b22421c84ed3" +checksum = "f729a06d0257428fd9d997ac8db7e1407bd0262ab016b085b36bc52957c0052d" dependencies = [ "cc", "pkg-config", diff --git a/Cargo.toml b/Cargo.toml index c931a0835d..c3a92fe5e8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -45,7 +45,7 @@ java-properties = "2.0.0" lazy_static = "1.4.0" libc = "0.2.139" log = { version = "0.4.17", features = ["std"] } -objectstore-client = { version = "0.1.12" , default-features = false, features = ["native-tls"] } +objectstore-client = { version = "0.3.2", default-features = false, features = ["native-tls"] } open = "3.2.0" parking_lot = "0.12.1" percent-encoding = "2.2.0" diff --git a/src/commands/snapshots/upload.rs b/src/commands/snapshots/upload.rs index fa1d3ad404..ef119e0d38 100644 --- a/src/commands/snapshots/upload.rs +++ b/src/commands/snapshots/upload.rs @@ -385,6 +385,18 @@ fn is_image_file(path: &Path) -> bool { .unwrap_or(false) } +/// Returns the MIME type of an image based on its file extension. +fn image_content_type(path: &Path) -> Option<&'static str> { + let ext = path.extension()?.to_str()?; + if ext.eq_ignore_ascii_case("png") { + Some("image/png") + } else if ext.eq_ignore_ascii_case("jpg") || ext.eq_ignore_ascii_case("jpeg") { + Some("image/jpeg") + } else { + None + } +} + /// Reads the companion JSON sidecar for an image, if it exists. /// /// For an image at `path/to/button.png`, looks for `path/to/button.json`. @@ -425,11 +437,9 @@ fn upload_images( let expiration = ExpirationPolicy::from_str(&options.objectstore.expiration_policy) .context("Failed to parse expiration policy from upload options")?; - let mut builder = ClientBuilder::new(options.objectstore.url); - if let Some(token) = options.objectstore.auth_token { - builder = builder.token(token.expose_secret().to_owned()); - } - let client = builder + let auth_token = options.objectstore.auth_token.as_ref(); + let client = ClientBuilder::new(options.objectstore.url) + .token(auth_token.map(|token| token.expose_secret().as_str())) .configure_reqwest(|r| r.connect_timeout(Duration::from_secs(10))) .build()?; @@ -445,7 +455,9 @@ fn upload_images( let project_id = find_scope("project").context("Missing project in UploadOptions scope")?; debug!("Using objectstore usecase {}", options.objectstore.usecase); - let mut scope = Usecase::new(&options.objectstore.usecase).scope(); + let mut scope = Usecase::new(&options.objectstore.usecase) + .context("Invalid objectstore usecase in upload options")? + .scope(); for (key, value) in scopes { scope = scope.push(&key, value); } @@ -538,12 +550,16 @@ fn upload_images( if upload_count > 0 { let mut many_builder = session.many(); for prepared in missing_uploads { - many_builder = many_builder.push( - session - .put_path(prepared.path.clone()) - .key(&prepared.key) - .expiration_policy(expiration), - ); + // Images are already compressed, so skip client-side compression. + let mut put = session + .put_path(prepared.path.clone()) + .key(&prepared.key) + .expiration_policy(expiration) + .compress(None); + if let Some(content_type) = image_content_type(&prepared.path) { + put = put.content_type(content_type); + } + many_builder = many_builder.push(put); } let result = diff --git a/tests/integration/snapshots.rs b/tests/integration/snapshots.rs index e00d5825ab..a2fa2b0441 100644 --- a/tests/integration/snapshots.rs +++ b/tests/integration/snapshots.rs @@ -144,32 +144,35 @@ fn command_snapshots_upload_uses_server_usecase( let image = std::fs::read("tests/integration/_fixtures/snapshots/snapshot.png").unwrap(); let hash = format!("{:x}", Sha256::digest(image)); let batch_path = format!("/proxy/v1/objects:batch/{expected_usecase}/org=1;project=2/"); - let objectstore_mocks: Vec<_> = [("head", 404), ("insert", 200)] - .into_iter() - .map(|(operation, status)| { - objectstore - .mock("POST", batch_path.as_str()) - .match_header("x-os-auth", "Bearer objectstore-token") - .match_body(mockito::Matcher::AllOf(vec![ - mockito::Matcher::Regex(format!( - "x-sn-batch-operation-kind: {operation}\\r\\n" - )), - mockito::Matcher::Regex(format!( - "x-sn-batch-operation-key: 1%2F2%2F{hash}\\r\\n" - )), - ])) - .with_header("content-type", "multipart/form-data; boundary=response") - .with_body(format!( - "--response\r\n\ - Content-Disposition: form-data; name=\"part\"\r\n\ - x-sn-batch-operation-index: 0\r\n\ - x-sn-batch-operation-status: {status}\r\n\ - \r\n\r\n--response--\r\n" - )) - .expect(1) - .create() - }) - .collect(); + let key = format!("1/2/{hash}"); + let objectstore_mocks: Vec<_> = [ + (404, format!(r#"\{{"kind":"head","key":"{key}"\}}"#)), + // Images are sent uncompressed with their content type. + ( + 201, + format!( + r#"\{{"kind":"insert","key":"{key}","metadata":\{{"content_type":"image/png","expiration":"# + ), + ), + ] + .into_iter() + .map(|(status, descriptor)| { + objectstore + .mock("POST", batch_path.as_str()) + .match_header("x-os-auth", "Bearer objectstore-token") + .match_body(mockito::Matcher::Regex(descriptor)) + .with_header("content-type", "multipart/form-data; boundary=response") + .with_body(format!( + "--response\r\n\ + Content-Disposition: form-data; name=\"0\"\r\n\ + \r\n\ + {{\"status\":{status},\"key\":\"{key}\"}}\r\n\ + --response--\r\n" + )) + .expect(1) + .create() + }) + .collect(); let mut upload_options = json!({ "objectstore": { "url": format!("{}/proxy", objectstore.url()),