From 9e2ce439349a36340a72f2d8b3c1fc952346873b Mon Sep 17 00:00:00 2001 From: jiuker Date: Wed, 12 Aug 2026 11:32:23 +0800 Subject: [PATCH 1/4] fix: Windows large file support and binary-safe downloads Fixes two Windows (LLP64) issues reported via the VCPKG port: binary file corruption on download and 32-bit 'long' overflow for files larger than 2 GiB. Binary-safe downloads: - DownloadObject: open the temp output stream with std::ios::binary. On Windows the default text mode translates newlines and treats 0x1A as EOF, corrupting downloaded binary content (e.g. zip archives). Large file support (LLP64: 'long' is strictly 32-bit on Windows): - SelectResult (types.h) and the parser in select.cc: bytes_scanned / bytes_processed / bytes_returned switch from 'long int' to 'long long', and std::stol to std::stoll - StatObject (baseclient.cc): parse the content-length header with std::stoll so resp.size (size_t) is not truncated above 2 GiB - ComposeSource (args.h/args.cc): object_size_ member and its assignment switch from 'long' to 'long long' so composing sources larger than 2 GiB does not overflow No behavioral change on LP64 platforms (Linux/macOS), where 'long' is already 64-bit. Files changed: include/miniocpp/args.h, include/miniocpp/types.h, src/args.cc, src/baseclient.cc, src/client.cc, src/select.cc --- include/miniocpp/args.h | 2 +- include/miniocpp/types.h | 10 +++++----- src/args.cc | 2 +- src/baseclient.cc | 2 +- src/client.cc | 3 ++- src/select.cc | 12 ++++++------ 6 files changed, 16 insertions(+), 15 deletions(-) diff --git a/include/miniocpp/args.h b/include/miniocpp/args.h index 19870928..9d4716c7 100644 --- a/include/miniocpp/args.h +++ b/include/miniocpp/args.h @@ -382,7 +382,7 @@ struct ComposeSource : public ObjectConditionalReadArgs { utils::Multimap Headers() const; private: - long object_size_ = -1; + long long object_size_ = -1; utils::Multimap headers_; }; // struct ComposeSource diff --git a/include/miniocpp/types.h b/include/miniocpp/types.h index 5be1ae42..8266c88f 100644 --- a/include/miniocpp/types.h +++ b/include/miniocpp/types.h @@ -293,17 +293,17 @@ struct SelectRequest { struct SelectResult { error::Error err = error::SUCCESS; bool ended = false; - long int bytes_scanned = -1; - long int bytes_processed = -1; - long int bytes_returned = -1; + long long bytes_scanned = -1; + long long bytes_processed = -1; + long long bytes_returned = -1; std::string records; SelectResult() : ended(true) {} explicit SelectResult(error::Error err) : err(std::move(err)), ended(true) {} - SelectResult(long int bytes_scanned, long int bytes_processed, - long int bytes_returned) + SelectResult(long long bytes_scanned, long long bytes_processed, + long long bytes_returned) : bytes_scanned(bytes_scanned), bytes_processed(bytes_processed), bytes_returned(bytes_returned) {} diff --git a/src/args.cc b/src/args.cc index deb8e022..66b463a0 100644 --- a/src/args.cc +++ b/src/args.cc @@ -381,7 +381,7 @@ error::Error ComposeSource::BuildHeaders(size_t object_size, } } - object_size_ = static_cast(object_size); + object_size_ = static_cast(object_size); headers_ = CopyHeaders(); if (!headers_.Contains("x-amz-copy-source-if-match")) { headers_.Add("x-amz-copy-source-if-match", etag); diff --git a/src/baseclient.cc b/src/baseclient.cc index 492f5014..1196cf46 100644 --- a/src/baseclient.cc +++ b/src/baseclient.cc @@ -2049,7 +2049,7 @@ Result BaseClient::StatObject(StatObjectArgs args) { resp.etag = utils::Trim(response->headers.GetFront("etag"), '"'); std::string value = response->headers.GetFront("content-length"); - if (!value.empty()) resp.size = std::stol(value); + if (!value.empty()) resp.size = std::stoll(value); value = response->headers.GetFront("last-modified"); if (!value.empty()) { diff --git a/src/client.cc b/src/client.cc index c1045d74..433305e3 100644 --- a/src/client.cc +++ b/src/client.cc @@ -1165,7 +1165,8 @@ Result Client::DownloadObject(DownloadObjectArgs args) { std::string temp_filename = args.filename + "." + curlpp::escape(etag) + ".part.minio"; - std::ofstream fout(temp_filename, std::ios::trunc | std::ios::out); + std::ofstream fout(temp_filename, + std::ios::trunc | std::ios::out | std::ios::binary); if (!fout.is_open()) { return error::make("unable to open file " + temp_filename); diff --git a/src/select.cc b/src/select.cc index ce6c4db4..8cdbea65 100644 --- a/src/select.cc +++ b/src/select.cc @@ -195,21 +195,21 @@ bool SelectHandler::process(const http::DataFunctionArgs& /* args */, auto root = xdoc.select_node(xpath.c_str()); pugi::xpath_node text; std::string value; - long int bytes_scanned = -1; - long int bytes_processed = -1; - long int bytes_returned = -1; + long long bytes_scanned = -1; + long long bytes_processed = -1; + long long bytes_returned = -1; text = root.node().select_node("BytesScanned/text()"); value = text.node().value(); - if (!value.empty()) bytes_scanned = std::stol(value); + if (!value.empty()) bytes_scanned = std::stoll(value); text = root.node().select_node("BytesProcessed/text()"); value = text.node().value(); - if (!value.empty()) bytes_processed = std::stol(value); + if (!value.empty()) bytes_processed = std::stoll(value); text = root.node().select_node("BytesReturned/text()"); value = text.node().value(); - if (!value.empty()) bytes_returned = std::stol(value); + if (!value.empty()) bytes_returned = std::stoll(value); cont = result_func_( SelectResult(bytes_scanned, bytes_processed, bytes_returned)); From a788fe326776f6a7573af2c4e3194fa818641808 Mon Sep 17 00:00:00 2001 From: jiuker Date: Thu, 13 Aug 2026 15:38:41 +0800 Subject: [PATCH 2/4] apply suggestion apply suggestion --- include/miniocpp/args.h | 2 +- include/miniocpp/types.h | 12 +++++++----- src/args.cc | 6 +++--- src/select.cc | 6 +++--- 4 files changed, 14 insertions(+), 12 deletions(-) diff --git a/include/miniocpp/args.h b/include/miniocpp/args.h index 9d4716c7..ded4a4fb 100644 --- a/include/miniocpp/args.h +++ b/include/miniocpp/args.h @@ -382,7 +382,7 @@ struct ComposeSource : public ObjectConditionalReadArgs { utils::Multimap Headers() const; private: - long long object_size_ = -1; + std::optional object_size_; utils::Multimap headers_; }; // struct ComposeSource diff --git a/include/miniocpp/types.h b/include/miniocpp/types.h index 8266c88f..98e2910d 100644 --- a/include/miniocpp/types.h +++ b/include/miniocpp/types.h @@ -25,6 +25,7 @@ #include #include #include +#include #include #include #include @@ -293,17 +294,18 @@ struct SelectRequest { struct SelectResult { error::Error err = error::SUCCESS; bool ended = false; - long long bytes_scanned = -1; - long long bytes_processed = -1; - long long bytes_returned = -1; + std::optional bytes_scanned; + std::optional bytes_processed; + std::optional bytes_returned; std::string records; SelectResult() : ended(true) {} explicit SelectResult(error::Error err) : err(std::move(err)), ended(true) {} - SelectResult(long long bytes_scanned, long long bytes_processed, - long long bytes_returned) + SelectResult(std::optional bytes_scanned, + std::optional bytes_processed, + std::optional bytes_returned) : bytes_scanned(bytes_scanned), bytes_processed(bytes_processed), bytes_returned(bytes_returned) {} diff --git a/src/args.cc b/src/args.cc index 66b463a0..e4a89eb8 100644 --- a/src/args.cc +++ b/src/args.cc @@ -381,7 +381,7 @@ error::Error ComposeSource::BuildHeaders(size_t object_size, } } - object_size_ = static_cast(object_size); + object_size_ = object_size; headers_ = CopyHeaders(); if (!headers_.Contains("x-amz-copy-source-if-match")) { headers_.Add("x-amz-copy-source-if-match", etag); @@ -391,14 +391,14 @@ error::Error ComposeSource::BuildHeaders(size_t object_size, } size_t ComposeSource::ObjectSize() const { - if (object_size_ == -1) { + if (!object_size_.has_value()) { std::cerr << "ABORT: ComposeSource::BuildHeaders() must be called prior to " "this method invocation. This should not happen." << std::endl; std::terminate(); } - return object_size_; + return *object_size_; } utils::Multimap ComposeSource::Headers() const { diff --git a/src/select.cc b/src/select.cc index 8cdbea65..6851b35f 100644 --- a/src/select.cc +++ b/src/select.cc @@ -195,9 +195,9 @@ bool SelectHandler::process(const http::DataFunctionArgs& /* args */, auto root = xdoc.select_node(xpath.c_str()); pugi::xpath_node text; std::string value; - long long bytes_scanned = -1; - long long bytes_processed = -1; - long long bytes_returned = -1; + std::optional bytes_scanned; + std::optional bytes_processed; + std::optional bytes_returned; text = root.node().select_node("BytesScanned/text()"); value = text.node().value(); From 693c21c4d5c06357ebb730d4bc8b6e604273e6c3 Mon Sep 17 00:00:00 2001 From: jiuker Date: Thu, 13 Aug 2026 15:43:52 +0800 Subject: [PATCH 3/4] add test --- tests/tests.cc | 92 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 92 insertions(+) diff --git a/tests/tests.cc b/tests/tests.cc index 7d4de3b6..720b56e0 100644 --- a/tests/tests.cc +++ b/tests/tests.cc @@ -22,6 +22,7 @@ #include #include #include +#include #include using minio::Result; @@ -38,6 +39,7 @@ using minio::Result; #include #include #include +#include #include #include #include @@ -102,6 +104,43 @@ std::string RandBucketName() { std::string RandObjectName() { return RandomString(charset, 8); } +std::string PutUint32BigEndian(unsigned int v) { + std::string s(4, '\0'); + s[0] = static_cast((v >> 24) & 0xFF); + s[1] = static_cast((v >> 16) & 0xFF); + s[2] = static_cast((v >> 8) & 0xFF); + s[3] = static_cast(v & 0xFF); + return s; +} + +// Build a single S3 Select protocol frame (prelude + prelude CRC + headers +// + payload + message CRC) for the given event headers and XML payload. +std::string MakeSelectFrame(const std::map& headers, + const std::string& payload) { + std::string headerdata; + for (const auto& [name, value] : headers) { + headerdata += static_cast(name.length()); + headerdata += name; + headerdata += static_cast(7); // header value type: string + headerdata += static_cast((value.length() >> 8) & 0xFF); + headerdata += static_cast(value.length() & 0xFF); + headerdata += value; + } + headerdata += static_cast(0); // header terminator + + std::string data = headerdata + payload; + unsigned int total_length = 16 + static_cast(data.length()); + std::string prelude = + PutUint32BigEndian(total_length) + + PutUint32BigEndian(static_cast(headerdata.length())); + std::string prelude_crc = + PutUint32BigEndian(static_cast(utils::CRC32(prelude))); + std::string message = prelude + prelude_crc + data; + std::string message_crc = + PutUint32BigEndian(static_cast(utils::CRC32(message))); + return message + message_crc; +} + struct MakeBucketError : public std::runtime_error { MakeBucketError(std::string err) : runtime_error(err) {} }; @@ -1537,6 +1576,58 @@ class Tests { RemoveObject(bucket_name_, object_name); } } // TestAsyncOperations + + // Regression test for SelectHandler Stats metric parsing: metrics larger + // than INT32_MAX must round-trip as exact long long values (any fallback + // to std::stol on 32-bit Windows LLP64 would truncate them). Uses a + // synthetic Stats event frame, independent of large objects. + void SelectStatsMetrics() { + std::cout << "SelectStatsMetrics()" << std::endl; + + const long long scanned = 5000000000LL; + const long long processed = 6000000000LL; + const long long returned = 7000000000LL; + + std::map headers = { + {":message-type", "event"}, + {":event-type", "Stats"}, + }; + std::string payload = "" + std::to_string(scanned) + + "" + + std::to_string(processed) + + "" + + std::to_string(returned) + ""; + + bool stats_delivered = false; + minio::s3::SelectHandler handler( + [&](minio::s3::SelectResult result) -> bool { + if (result.err) { + throw std::runtime_error("SelectStatsMetrics(): " + + result.err.String()); + } + if (!result.bytes_scanned.has_value() || + *result.bytes_scanned != scanned || + !result.bytes_processed.has_value() || + *result.bytes_processed != processed || + !result.bytes_returned.has_value() || + *result.bytes_returned != returned) { + throw std::runtime_error( + "SelectStatsMetrics(): unexpected metrics"); + } + stats_delivered = true; + return true; + }); + + minio::http::DataFunctionArgs args; + args.datachunk = MakeSelectFrame(headers, payload); + if (!handler.DataFunction(args)) { + throw std::runtime_error("SelectStatsMetrics(): DataFunction failed"); + } + if (!stats_delivered) { + throw std::runtime_error( + "SelectStatsMetrics(): Stats result was not delivered"); + } + } }; // class Tests int main(int /*argc*/, char* /*argv*/[]) { @@ -1596,6 +1687,7 @@ int main(int /*argc*/, char* /*argv*/[]) { tests.SelectObjectContent(); tests.ListenBucketNotification(); tests.TestAsyncOperations(); + tests.SelectStatsMetrics(); return EXIT_SUCCESS; } From 1d9fe92c66cd3238fa6c2237461524956122c91e Mon Sep 17 00:00:00 2001 From: jiuker Date: Thu, 13 Aug 2026 15:44:39 +0800 Subject: [PATCH 4/4] Update tests.cc --- tests/tests.cc | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/tests.cc b/tests/tests.cc index 720b56e0..ae854268 100644 --- a/tests/tests.cc +++ b/tests/tests.cc @@ -133,11 +133,11 @@ std::string MakeSelectFrame(const std::map& headers, std::string prelude = PutUint32BigEndian(total_length) + PutUint32BigEndian(static_cast(headerdata.length())); - std::string prelude_crc = - PutUint32BigEndian(static_cast(utils::CRC32(prelude))); + std::string prelude_crc = PutUint32BigEndian( + static_cast(minio::utils::CRC32(prelude))); std::string message = prelude + prelude_crc + data; - std::string message_crc = - PutUint32BigEndian(static_cast(utils::CRC32(message))); + std::string message_crc = PutUint32BigEndian( + static_cast(minio::utils::CRC32(message))); return message + message_crc; }