From 280fc8c89f857610bfe69ec0001b9d3a2994a03c Mon Sep 17 00:00:00 2001 From: Ionite Date: Sun, 17 Dec 2023 18:02:58 -0500 Subject: [PATCH] Add animated webp rendering --- Avalonia.Gif/Avalonia.Gif.csproj | 1 + Avalonia.Gif/GifImage.cs | 9 +- Avalonia.Gif/GifInstance.cs | 2 +- Avalonia.Gif/IGifInstance.cs | 15 ++ Avalonia.Gif/WebpInstance.cs | 180 ++++++++++++++++++ .../Base/InferenceGenerationViewModelBase.cs | 8 +- .../InferenceImageToVideoViewModel.cs | 2 +- .../Models/Database/LocalImageFile.cs | 2 +- 8 files changed, 208 insertions(+), 11 deletions(-) create mode 100644 Avalonia.Gif/IGifInstance.cs create mode 100644 Avalonia.Gif/WebpInstance.cs diff --git a/Avalonia.Gif/Avalonia.Gif.csproj b/Avalonia.Gif/Avalonia.Gif.csproj index 2d72f153..49c6015e 100644 --- a/Avalonia.Gif/Avalonia.Gif.csproj +++ b/Avalonia.Gif/Avalonia.Gif.csproj @@ -11,6 +11,7 @@ + diff --git a/Avalonia.Gif/GifImage.cs b/Avalonia.Gif/GifImage.cs index d53cbd19..0ce8dca8 100644 --- a/Avalonia.Gif/GifImage.cs +++ b/Avalonia.Gif/GifImage.cs @@ -32,7 +32,7 @@ namespace Avalonia.Gif IterationCount >("IterationCount", IterationCount.Infinite); - private GifInstance? _gifInstance; + private IGifInstance? _gifInstance; public static readonly StyledProperty StretchDirectionProperty = AvaloniaProperty.Register< GifImage, @@ -166,7 +166,7 @@ namespace Avalonia.Gif { private TimeSpan _animationElapsed; private TimeSpan? _lastServerTime; - private GifInstance? _currentInstance; + private IGifInstance? _currentInstance; private bool _running; public static readonly object StopMessage = new(), @@ -184,7 +184,7 @@ namespace Avalonia.Gif { _running = false; } - else if (message is GifInstance instance) + else if (message is IGifInstance instance) { _currentInstance?.Dispose(); _currentInstance = instance; @@ -288,7 +288,8 @@ namespace Avalonia.Gif private void UpdateGifInstance(object source) { _gifInstance?.Dispose(); - _gifInstance = new GifInstance(source); + _gifInstance = new WebpInstance(source); + // _gifInstance = new GifInstance(source); _gifInstance.IterationCount = IterationCount; _customVisual?.SendHandlerMessage(_gifInstance); } diff --git a/Avalonia.Gif/GifInstance.cs b/Avalonia.Gif/GifInstance.cs index 30e002d1..b97badaa 100644 --- a/Avalonia.Gif/GifInstance.cs +++ b/Avalonia.Gif/GifInstance.cs @@ -11,7 +11,7 @@ using Avalonia.Platform; namespace Avalonia.Gif { - public class GifInstance : IDisposable + public class GifInstance : IGifInstance { public IterationCount IterationCount { get; set; } public bool AutoStart { get; private set; } = true; diff --git a/Avalonia.Gif/IGifInstance.cs b/Avalonia.Gif/IGifInstance.cs new file mode 100644 index 00000000..667f9163 --- /dev/null +++ b/Avalonia.Gif/IGifInstance.cs @@ -0,0 +1,15 @@ +using Avalonia.Animation; +using Avalonia.Media.Imaging; + +namespace Avalonia.Gif; + +public interface IGifInstance : IDisposable +{ + IterationCount IterationCount { get; set; } + bool AutoStart { get; } + CancellationTokenSource CurrentCts { get; } + int GifFrameCount { get; } + PixelSize GifPixelSize { get; } + bool IsDisposed { get; } + WriteableBitmap? ProcessFrameTime(TimeSpan stopwatchElapsed); +} diff --git a/Avalonia.Gif/WebpInstance.cs b/Avalonia.Gif/WebpInstance.cs new file mode 100644 index 00000000..b9256934 --- /dev/null +++ b/Avalonia.Gif/WebpInstance.cs @@ -0,0 +1,180 @@ +using Avalonia.Animation; +using Avalonia.Media.Imaging; +using Avalonia.Platform; +using SkiaSharp; + +namespace Avalonia.Gif; + +public class WebpInstance : IGifInstance +{ + public IterationCount IterationCount { get; set; } + public bool AutoStart { get; private set; } = true; + + private readonly WriteableBitmap? _targetBitmap; + private TimeSpan _totalTime; + private readonly List _frameTimes; + private uint _iterationCount; + private int _currentFrameIndex; + + private SKCodec? _codec; + + public CancellationTokenSource CurrentCts { get; } + + internal WebpInstance(object newValue) + : this( + newValue switch + { + Stream s => s, + Uri u => GetStreamFromUri(u), + string str => GetStreamFromString(str), + _ => throw new InvalidDataException("Unsupported source object") + } + ) { } + + public WebpInstance(string uri) + : this(GetStreamFromString(uri)) { } + + public WebpInstance(Uri uri) + : this(GetStreamFromUri(uri)) { } + + public WebpInstance(Stream currentStream) + { + if (!currentStream.CanSeek) + throw new InvalidDataException("The provided stream is not seekable."); + + if (!currentStream.CanRead) + throw new InvalidOperationException("Can't read the stream provided."); + + currentStream.Seek(0, SeekOrigin.Begin); + + CurrentCts = new CancellationTokenSource(); + + var managedStream = new SKManagedStream(currentStream); + _codec = SKCodec.Create(managedStream); + + var pixSize = new PixelSize(_codec.Info.Width, _codec.Info.Height); + + _targetBitmap = new WriteableBitmap(pixSize, new Vector(96, 96), PixelFormat.Bgra8888, AlphaFormat.Opaque); + GifPixelSize = pixSize; + + _totalTime = TimeSpan.Zero; + + _frameTimes = _codec + .FrameInfo + .Select(frame => + { + _totalTime = _totalTime.Add(TimeSpan.FromMilliseconds(frame.Duration)); + return _totalTime; + }) + .ToList(); + + RenderFrame(_codec, _targetBitmap, 0); + } + + private static void RenderFrame(SKCodec codec, WriteableBitmap targetBitmap, int index) + { + codec.GetFrameInfo(index, out var frameInfo); + + var info = new SKImageInfo(codec.Info.Width, codec.Info.Height); + var decodeInfo = info.WithAlphaType(frameInfo.AlphaType); + + using var frameBuffer = targetBitmap.Lock(); + + var result = codec.GetPixels(decodeInfo, frameBuffer.Address, new SKCodecOptions(index)); + + if (result != SKCodecResult.Success) + throw new InvalidDataException($"Could not decode frame {index} of {codec.FrameCount}."); + } + + private static void RenderFrame(SKCodec codec, WriteableBitmap targetBitmap, int index, int priorIndex) + { + codec.GetFrameInfo(index, out var frameInfo); + + var info = new SKImageInfo(codec.Info.Width, codec.Info.Height); + var decodeInfo = info.WithAlphaType(frameInfo.AlphaType); + + using var frameBuffer = targetBitmap.Lock(); + + var result = codec.GetPixels(decodeInfo, frameBuffer.Address, new SKCodecOptions(index, priorIndex)); + + if (result != SKCodecResult.Success) + throw new InvalidDataException($"Could not decode frame {index} of {codec.FrameCount}."); + } + + private static Stream GetStreamFromString(string str) + { + if (!Uri.TryCreate(str, UriKind.RelativeOrAbsolute, out var res)) + { + throw new InvalidCastException("The string provided can't be converted to URI."); + } + + return GetStreamFromUri(res); + } + + private static Stream GetStreamFromUri(Uri uri) + { + var uriString = uri.OriginalString.Trim(); + + if (!uriString.StartsWith("resm") && !uriString.StartsWith("avares")) + { + return new FileStream(uriString, FileMode.Open, FileAccess.Read); + } + + return AssetLoader.Open(uri); + } + + public int GifFrameCount => _frameTimes.Count; + + public PixelSize GifPixelSize { get; } + + public void Dispose() + { + IsDisposed = true; + CurrentCts.Cancel(); + _targetBitmap?.Dispose(); + _codec?.Dispose(); + } + + public bool IsDisposed { get; private set; } + + public WriteableBitmap? ProcessFrameTime(TimeSpan stopwatchElapsed) + { + if (!IterationCount.IsInfinite && _iterationCount > IterationCount.Value) + { + return null; + } + + if (CurrentCts.IsCancellationRequested || _targetBitmap is null) + { + return null; + } + + var elapsedTicks = stopwatchElapsed.Ticks; + var timeModulus = TimeSpan.FromTicks(elapsedTicks % _totalTime.Ticks); + var targetFrame = _frameTimes.FirstOrDefault(x => timeModulus < x); + var currentFrame = _frameTimes.IndexOf(targetFrame); + if (currentFrame == -1) + currentFrame = 0; + + if (_currentFrameIndex == currentFrame) + return _targetBitmap; + + _iterationCount = (uint)(elapsedTicks / _totalTime.Ticks); + + return ProcessFrameIndex(currentFrame); + } + + internal WriteableBitmap ProcessFrameIndex(int frameIndex) + { + if (_codec is null) + throw new InvalidOperationException("The codec is null."); + + if (_targetBitmap is null) + throw new InvalidOperationException("The target bitmap is null."); + + RenderFrame(_codec, _targetBitmap, frameIndex, _currentFrameIndex); + _currentFrameIndex = frameIndex; + + return _targetBitmap; + } +} diff --git a/StabilityMatrix.Avalonia/ViewModels/Base/InferenceGenerationViewModelBase.cs b/StabilityMatrix.Avalonia/ViewModels/Base/InferenceGenerationViewModelBase.cs index 51ea578b..99183592 100644 --- a/StabilityMatrix.Avalonia/ViewModels/Base/InferenceGenerationViewModelBase.cs +++ b/StabilityMatrix.Avalonia/ViewModels/Base/InferenceGenerationViewModelBase.cs @@ -409,14 +409,14 @@ public abstract partial class InferenceGenerationViewModelBase : InferenceTabVie ); // convert to gif - var inputStream = File.OpenRead(webpFilePath); + /*var inputStream = File.OpenRead(webpFilePath); var gifFilePath = webpFilePath.ToString().Replace(".webp", ".gif"); var outputStream = File.OpenWrite(gifFilePath); await GifConverter.ConvertAnimatedWebpToGifAsync(inputStream, outputStream); await inputStream.DisposeAsync(); await outputStream.FlushAsync(); - await outputStream.DisposeAsync(); + await outputStream.DisposeAsync();*/ // if (File.Exists(gifFilePath)) // { @@ -424,8 +424,8 @@ public abstract partial class InferenceGenerationViewModelBase : InferenceTabVie // File.Delete(webpFilePath); // } - outputImages.Add(new ImageSource(gifFilePath)); - EventManager.Instance.OnImageFileAdded(gifFilePath); + outputImages.Add(new ImageSource(webpFilePath)); + EventManager.Instance.OnImageFileAdded(webpFilePath); } else { diff --git a/StabilityMatrix.Avalonia/ViewModels/Inference/InferenceImageToVideoViewModel.cs b/StabilityMatrix.Avalonia/ViewModels/Inference/InferenceImageToVideoViewModel.cs index b1e6d838..4be708a7 100644 --- a/StabilityMatrix.Avalonia/ViewModels/Inference/InferenceImageToVideoViewModel.cs +++ b/StabilityMatrix.Avalonia/ViewModels/Inference/InferenceImageToVideoViewModel.cs @@ -130,7 +130,7 @@ public partial class InferenceImageToVideoViewModel : InferenceGenerationViewMod private void OnImageFileAdded(object? sender, FilePath e) { - if (!e.Extension.Contains("gif")) + if (!e.Extension.Equals(".webp", StringComparison.OrdinalIgnoreCase)) return; OutputUri = e; diff --git a/StabilityMatrix.Core/Models/Database/LocalImageFile.cs b/StabilityMatrix.Core/Models/Database/LocalImageFile.cs index 3f327f28..a050b6c6 100644 --- a/StabilityMatrix.Core/Models/Database/LocalImageFile.cs +++ b/StabilityMatrix.Core/Models/Database/LocalImageFile.cs @@ -104,5 +104,5 @@ public record LocalImageFile }; } - public static readonly HashSet SupportedImageExtensions = [".png", ".jpg", ".jpeg", ".gif"]; + public static readonly HashSet SupportedImageExtensions = [".png", ".jpg", ".jpeg", ".gif", ".webp"]; }