Browse Source

Use smaller blake3 buffer in line with https://github.com/zeebo/blake3

pull/324/head
Ionite 12 months ago
parent
commit
3689ea3229
No known key found for this signature in database
  1. 42
      StabilityMatrix.Core/Helper/FileHash.cs

42
StabilityMatrix.Core/Helper/FileHash.cs

@ -7,16 +7,21 @@ namespace StabilityMatrix.Core.Helper;
public static class FileHash
{
public static async Task<string> GetHashAsync(HashAlgorithm hashAlgorithm, Stream stream, byte[] buffer, Action<ulong>? progress = default)
public static async Task<string> GetHashAsync(
HashAlgorithm hashAlgorithm,
Stream stream,
byte[] buffer,
Action<ulong>? progress = default
)
{
ulong totalBytesRead = 0;
using (hashAlgorithm)
{
int bytesRead;
while ((bytesRead = await stream.ReadAsync(buffer)) != 0)
while ((bytesRead = await stream.ReadAsync(buffer).ConfigureAwait(false)) != 0)
{
totalBytesRead += (ulong) bytesRead;
totalBytesRead += (ulong)bytesRead;
hashAlgorithm.TransformBlock(buffer, 0, bytesRead, null, 0);
progress?.Invoke(totalBytesRead);
}
@ -39,22 +44,27 @@ public static class FileHash
var totalBytes = Convert.ToUInt64(new FileInfo(filePath).Length);
var shared = ArrayPool<byte>.Shared;
var buffer = shared.Rent((int) FileTransfers.GetBufferSize(totalBytes));
var buffer = shared.Rent((int)FileTransfers.GetBufferSize(totalBytes));
try
{
await using var stream = File.OpenRead(filePath);
var hash = await GetHashAsync(SHA256.Create(), stream, buffer, totalBytesRead =>
var hash = await GetHashAsync(
SHA256.Create(),
stream,
buffer,
totalBytesRead =>
{
progress?.Report(new ProgressReport(totalBytesRead, totalBytes, type: ProgressType.Hashing));
}).ConfigureAwait(false);
}
)
.ConfigureAwait(false);
return hash;
}
finally
{
shared.Return(buffer);
}
}
public static async Task<string> GetBlake3Async(string filePath, IProgress<ProgressReport>? progress = default)
@ -67,19 +77,19 @@ public static class FileHash
var totalBytes = Convert.ToUInt64(new FileInfo(filePath).Length);
var readBytes = 0ul;
var shared = ArrayPool<byte>.Shared;
var buffer = shared.Rent((int) FileTransfers.GetBufferSize(totalBytes));
var buffer = shared.Rent(GetBufferSize(totalBytes));
try
{
await using var stream = File.OpenRead(filePath);
using var hasher = Hasher.New();
while (true)
{
var bytesRead = await stream.ReadAsync(buffer);
var bytesRead = await stream.ReadAsync(buffer).ConfigureAwait(false);
if (bytesRead == 0)
{
break;
}
readBytes += (ulong) bytesRead;
readBytes += (ulong)bytesRead;
hasher.Update(buffer.AsSpan(0, bytesRead));
progress?.Report(new ProgressReport(readBytes, totalBytes));
}
@ -90,4 +100,16 @@ public static class FileHash
shared.Return(buffer);
}
}
/// <summary>
/// Determines suitable buffer size for hashing based on stream length.
/// </summary>
private static int GetBufferSize(ulong totalBytes) =>
totalBytes switch
{
< Size.MiB => 8 * (int)Size.KiB,
< 500 * Size.MiB => 16 * (int)Size.KiB,
< Size.GiB => 32 * (int)Size.KiB,
_ => 64 * (int)Size.KiB
};
}

Loading…
Cancel
Save