Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 22 additions & 0 deletions google-cloud-storage/acceptance/storage/file_test.rb
Original file line number Diff line number Diff line change
Expand Up @@ -476,6 +476,28 @@
end
end

it "should upload and download an unlinked tempfile" do
begin
data = "hello world"
tmpfile = Tempfile.new "unlinked_test"
tmpfile.write data
tmpfile.rewind
tmpfile.unlink

uploaded = bucket.create_file tmpfile, "uploaded/unlinked-file.txt"
_(uploaded.name).must_equal "uploaded/unlinked-file.txt"

downloadio = StringIO.new
downloaded = uploaded.download downloadio
_(downloaded).must_be_kind_of StringIO

downloaded_data = downloaded.string
_(downloaded_data).must_equal data
ensure
uploaded.delete if uploaded
end
end

it "should download and verify when Content-Encoding gzip response header with skip_decompress" do
bucket = bucket_public
file = bucket.file bucket_public_file_gzip
Expand Down
34 changes: 18 additions & 16 deletions google-cloud-storage/lib/google/cloud/storage/file/verifier.rb
Original file line number Diff line number Diff line change
Expand Up @@ -63,33 +63,35 @@ def self.crc32c_for local_file
# Computes a base64-encoded digest for a local file or IO stream.
#
# This method handles two types of inputs for `local_file`:
# 1. A file path (String or Pathname): It efficiently streams the file

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why are we removing this?

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

we are swapping the order of Case 1 and Case 2 to match the updated logic below. We need to check if the object is an IO-like stream (i.e., responds to :read) before checking if it's a file path. If we check for the file path first, an unlinked Tempfile will still return true for :to_path, but attempting to open that path will raise an Errno::ENOENT error. Checking if we can read it directly (:read) first fixes this.

# to compute the digest without loading the entire file into memory.
# 2. An IO-like stream (e.g., File, StringIO): It reads the stream's
# content to compute the digest. The stream is rewound before and after
# 1. An IO-like stream (e.g., File, Tempfile, StringIO): It reads the
# stream's content in chunks to compute the digest without loading
# the entire file into memory. The stream is rewound before and after
# reading to ensure its position is not permanently changed.
# 2. A file path (String or Pathname): It efficiently streams the file

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm not sure if this is true

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

reverted to previous version

# to compute the digest without loading the entire file into memory.
#
# @param local_file [String, Pathname, IO] The local file path or IO
# stream for which to compute the digest.
# @param digest_class [Class] The digest class to use for the
# calculation (e.g., `Digest::MD5`). It must respond to `.file` and
# `.base64digest`.
# calculation (e.g., `Digest::MD5`). It must respond to `.new`,
# `.file` and `.base64digest`.
#
# @return [String] The base64-encoded digest of the file's content.
#
def self._digest_for local_file, digest_class

if local_file.respond_to?(:to_path) || local_file.is_a?(String)
# Case 1: Input is a file path (String, Pathname, or object that responds to :to_path).
::File.open Pathname(local_file).to_path, "rb" do |f|
digest_class.file(f).base64digest
if local_file.respond_to?(:read)

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why are we removing existing code?

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

we are swapping the existing code to check for the :read capability (the IO-stream case) before falling back to the :to_path capability (the file path case).

# Case 1: Input is an open stream (File, StringIO, ActionDispatch::Http::UploadedFile, etc.).
local_file.rewind if local_file.respond_to?(:rewind)
digest = digest_class.new
while (chunk = local_file.read(16 * 1024))
digest.update(chunk)
end
else
# Case 2: Input is an open stream (File or StringIO).
local_file.rewind
digest = digest_class.base64digest local_file.read
local_file.rewind
digest
local_file.rewind if local_file.respond_to?(:rewind)
digest.base64digest
elsif local_file.respond_to?(:to_path) || local_file.is_a?(String)
# Case 2: Input is a file path (String, Pathname).
digest_class.file(Pathname(local_file).to_path).base64digest
end
end
end
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -116,9 +116,10 @@ def stub.list_buckets *args

# Stub the crc32c to match.
def file.crc32c
"AAAAAA=="
"ltgT7g=="
end


downloaded = file.download tmpfile
_(downloaded).must_be_kind_of Tempfile

Expand All @@ -144,7 +145,7 @@ def file.crc32c

# Stub the crc32c to match.
def file.crc32c
"AAAAAA=="
"ltgT7g=="
end

downloaded = file.download tmpfile
Expand All @@ -153,7 +154,7 @@ def file.crc32c

local_data = downloaded.read
local_crc32c = Digest::CRC32c.base64digest local_data
_(local_crc32c).must_equal "AAAAAA=="
_(local_crc32c).must_equal "ltgT7g=="

mock.verify
end
Expand Down
24 changes: 22 additions & 2 deletions google-cloud-storage/test/google/cloud/storage/verifier_test.rb
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,8 @@

describe Google::Cloud::Storage::File::Verifier, :mock_storage do
let(:file_contents) { "The quick brown fox jumps over the lazy dog." }
let(:md5_digest) { "1B2M2Y8AsgTpgAmY7PhCfg==" }
let(:crc32c_digest) { "AAAAAA==" }
let(:md5_digest) { "5NkJwpDQ+xygaP+t3yLL0A==" }
let(:crc32c_digest) { "GQCXsw==" }
let(:file_gapi) { Google::Apis::StorageV1::Object.from_json corrected_file_hash.to_json }
let(:file) { Google::Cloud::Storage::File.from_gapi file_gapi, OpenStruct.new }

Expand Down Expand Up @@ -51,6 +51,26 @@
end
end

it "calculates md5 digest for an unlinked file" do
Tempfile.open "google-cloud" do |tmpfile|
tmpfile.write file_contents
tmpfile.rewind
tmpfile.unlink
digest = Google::Cloud::Storage::File::Verifier.md5_for tmpfile
_(digest).must_equal md5_digest
end
end

it "calculates crc32c digest for an unlinked file" do
Tempfile.open "google-cloud" do |tmpfile|
tmpfile.write file_contents
tmpfile.rewind
tmpfile.unlink
digest = Google::Cloud::Storage::File::Verifier.crc32c_for tmpfile
_(digest).must_equal crc32c_digest
end
end

def corrected_file_hash
hash = random_file_hash("bucket", "file.ext")
hash["md5Hash"] = md5_digest
Expand Down
Loading