Browse Source

save/load img2vid and show webps in dialog thing

pull/438/head
JT 11 months ago
parent
commit
a8894f9631
  1. 5
      StabilityMatrix.Avalonia/Models/Inference/VideoOutputMethod.cs
  2. 19
      StabilityMatrix.Avalonia/ViewModels/Base/InferenceGenerationViewModelBase.cs
  3. 7
      StabilityMatrix.Avalonia/ViewModels/Inference/InferenceImageToVideoViewModel.cs
  4. 16
      StabilityMatrix.Avalonia/ViewModels/Inference/Video/SvdImgToVidConditioningViewModel.cs
  5. 20
      StabilityMatrix.Avalonia/ViewModels/Inference/Video/VideoOutputSettingsCardViewModel.cs
  6. 5
      StabilityMatrix.Avalonia/ViewModels/Settings/MainSettingsViewModel.cs
  7. 59
      StabilityMatrix.Avalonia/Views/Dialogs/ImageViewerDialog.axaml
  8. 48
      StabilityMatrix.Core/Helper/ImageMetadata.cs
  9. 44
      StabilityMatrix.Core/Models/Database/LocalImageFile.cs
  10. 9
      StabilityMatrix.Core/Models/GenerationParameters.cs

5
StabilityMatrix.Avalonia/Models/Inference/VideoOutputMethod.cs

@ -1,5 +1,8 @@
namespace StabilityMatrix.Avalonia.Models.Inference;
using System.Text.Json.Serialization;
namespace StabilityMatrix.Avalonia.Models.Inference;
[JsonConverter(typeof(JsonStringEnumConverter))]
public enum VideoOutputMethod
{
Fastest,

19
StabilityMatrix.Avalonia/ViewModels/Base/InferenceGenerationViewModelBase.cs

@ -5,6 +5,8 @@ using System.Diagnostics;
using System.Diagnostics.CodeAnalysis;
using System.IO;
using System.Linq;
using System.Management;
using System.Text.Json;
using System.Text.Json.Serialization;
using System.Threading;
using System.Threading.Tasks;
@ -12,6 +14,8 @@ using AsyncAwaitBestPractices;
using Avalonia.Controls.Notifications;
using Avalonia.Threading;
using CommunityToolkit.Mvvm.Input;
using ExifLibrary;
using MetadataExtractor.Formats.Exif;
using NLog;
using Refit;
using SkiaSharp;
@ -422,7 +426,20 @@ public abstract partial class InferenceGenerationViewModelBase
}
else if (comfyImage.FileName.EndsWith(".webp"))
{
var bytesWithMetadata = ImageMetadata.AddMetadataToWebp(imageArray, parameters);
var opts = new JsonSerializerOptions
{
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
Converters = { new JsonStringEnumConverter() }
};
var paramsJson = JsonSerializer.Serialize(parameters, opts);
var smProject = JsonSerializer.Serialize(project, opts);
var metadata = new Dictionary<ExifTag, string>
{
{ ExifTag.ImageDescription, paramsJson },
{ ExifTag.Software, smProject }
};
var bytesWithMetadata = ImageMetadata.AddMetadataToWebp(imageArray, metadata);
// Write using generated name
var filePath = await WriteOutputImageAsync(

7
StabilityMatrix.Avalonia/ViewModels/Inference/InferenceImageToVideoViewModel.cs

@ -1,7 +1,6 @@
using System;
using System.Collections.Generic;
using System.ComponentModel.DataAnnotations;
using System.Drawing;
using System.Linq;
using System.Text.Json.Nodes;
using System.Text.Json.Serialization;
@ -23,8 +22,6 @@ using StabilityMatrix.Core.Models.Api.Comfy.Nodes;
using StabilityMatrix.Core.Models.FileInterfaces;
using StabilityMatrix.Core.Services;
#pragma warning disable CS0657 // Not a valid attribute location for this declaration
namespace StabilityMatrix.Avalonia.ViewModels.Inference;
[View(typeof(InferenceImageToVideoView), persistent: true)]
@ -218,6 +215,8 @@ public partial class InferenceImageToVideoViewModel
{
SamplerCardViewModel.LoadStateFromParameters(parameters);
ModelCardViewModel.LoadStateFromParameters(parameters);
SvdImgToVidConditioningViewModel.LoadStateFromParameters(parameters);
VideoOutputSettingsCardViewModel.LoadStateFromParameters(parameters);
SeedCardViewModel.Seed = Convert.ToInt64(parameters.Seed);
}
@ -227,6 +226,8 @@ public partial class InferenceImageToVideoViewModel
{
parameters = SamplerCardViewModel.SaveStateToParameters(parameters);
parameters = ModelCardViewModel.SaveStateToParameters(parameters);
parameters = SvdImgToVidConditioningViewModel.SaveStateToParameters(parameters);
parameters = VideoOutputSettingsCardViewModel.SaveStateToParameters(parameters);
parameters.Seed = (ulong)SeedCardViewModel.Seed;

16
StabilityMatrix.Avalonia/ViewModels/Inference/Video/SvdImgToVidConditioningViewModel.cs

@ -44,13 +44,23 @@ public partial class SvdImgToVidConditioningViewModel
{
Width = parameters.Width;
Height = parameters.Height;
// TODO: add more metadata
NumFrames = parameters.FrameCount;
MotionBucketId = parameters.MotionBucketId;
Fps = parameters.Fps;
AugmentationLevel = parameters.AugmentationLevel;
MinCfg = parameters.MinCfg;
}
public GenerationParameters SaveStateToParameters(GenerationParameters parameters)
{
// TODO: add more metadata
return parameters with { Width = Width, Height = Height, };
return parameters with
{
FrameCount = NumFrames,
MotionBucketId = MotionBucketId,
Fps = Fps,
AugmentationLevel = AugmentationLevel,
MinCfg = MinCfg,
};
}
public void ApplyStep(ModuleApplyStepEventArgs e)

20
StabilityMatrix.Avalonia/ViewModels/Inference/Video/VideoOutputSettingsCardViewModel.cs

@ -37,13 +37,27 @@ public partial class VideoOutputSettingsCardViewModel
public void LoadStateFromParameters(GenerationParameters parameters)
{
// TODO
Fps = parameters.OutputFps;
Lossless = parameters.Lossless;
Quality = parameters.VideoQuality;
if (string.IsNullOrWhiteSpace(parameters.VideoOutputMethod))
return;
SelectedMethod = Enum.TryParse<VideoOutputMethod>(parameters.VideoOutputMethod, true, out var method)
? method
: VideoOutputMethod.Default;
}
public GenerationParameters SaveStateToParameters(GenerationParameters parameters)
{
// TODO
return parameters;
return parameters with
{
OutputFps = Fps,
Lossless = Lossless,
VideoQuality = Quality,
VideoOutputMethod = SelectedMethod.ToString(),
};
}
public void ApplyStep(ModuleApplyStepEventArgs e)

5
StabilityMatrix.Avalonia/ViewModels/Settings/MainSettingsViewModel.cs

@ -26,6 +26,7 @@ using CommunityToolkit.Mvvm.Input;
using DynamicData.Binding;
using ExifLibrary;
using FluentAvalonia.UI.Controls;
using MetadataExtractor.Formats.Exif;
using NLog;
using SkiaSharp;
using StabilityMatrix.Avalonia.Animations;
@ -709,10 +710,6 @@ public partial class MainSettingsViewModel : PageViewModelBase
if (files.Count == 0)
return;
var data = ImageMetadata.ReadTextChunkFromWebp(files[0].TryGetLocalPath(), ExifTag.ImageDescription);
return;
var metadata = ImageMetadata.ParseFile(files[0].TryGetLocalPath()!);
var textualTags = metadata.GetTextualData()?.ToArray();

59
StabilityMatrix.Avalonia/Views/Dialogs/ImageViewerDialog.axaml

@ -10,6 +10,9 @@
xmlns:mocks="clr-namespace:StabilityMatrix.Avalonia.DesignData"
xmlns:ui="clr-namespace:FluentAvalonia.UI.Controls;assembly=FluentAvalonia"
xmlns:vmDialogs="clr-namespace:StabilityMatrix.Avalonia.ViewModels.Dialogs"
xmlns:models="clr-namespace:StabilityMatrix.Avalonia.Models"
xmlns:gif="clr-namespace:Avalonia.Gif;assembly=Avalonia.Gif"
xmlns:input="clr-namespace:FluentAvalonia.UI.Input;assembly=FluentAvalonia"
HorizontalContentAlignment="Stretch"
VerticalContentAlignment="Stretch"
d:DataContext="{x:Static mocks:DesignData.ImageViewerViewModel}"
@ -46,24 +49,46 @@
HorizontalAlignment="Stretch"
VerticalAlignment="Stretch"
RowDefinitions="*,Auto">
<Grid.Resources>
<input:StandardUICommand
x:Key="CopyImageCommand"
Command="{Binding CopyImageCommand}"/>
</Grid.Resources>
<controls:AdvancedImageBox
Name="ImageBox"
Image="{Binding ImageSource.BitmapAsync^}"
SizeMode="Fit">
<controls:AdvancedImageBox.ContextFlyout>
<ui:FAMenuFlyout>
<ui:MenuFlyoutItem
x:Name="CopyMenuItem"
IsEnabled="{OnPlatform Windows=True, Default=False}"
Command="{Binding CopyImageCommand}"
CommandParameter="{Binding #ImageBox.Image}"
Text="Copy"
HotKey="Ctrl+C"
IconSource="Copy" />
</ui:FAMenuFlyout>
</controls:AdvancedImageBox.ContextFlyout>
</controls:AdvancedImageBox>
<ContentPresenter Grid.Row="0"
DataContext="{Binding ImageSource}"
Content="{Binding }">
<ContentPresenter.ContentTemplate>
<controls:DataTemplateSelector x:TypeArguments="models:ImageSourceTemplateType">
<DataTemplate x:Key="{x:Static models:ImageSourceTemplateType.WebpAnimation}" DataType="models:ImageSource">
<gif:GifImage
Stretch="Uniform"
SourceUri="{Binding LocalFile.FullPath}"/>
</DataTemplate>
<DataTemplate x:Key="{x:Static models:ImageSourceTemplateType.Image}" DataType="models:ImageSource">
<controls:AdvancedImageBox
Name="ImageBox"
CornerRadius="4"
Image="{Binding BitmapAsync^}"
SizeMode="Fit">
<controls:AdvancedImageBox.ContextFlyout>
<ui:FAMenuFlyout>
<ui:MenuFlyoutItem
x:Name="CopyMenuItem"
Command="{StaticResource CopyImageCommand}"
CommandParameter="{Binding $parent[controls:AdvancedImageBox].Image}"
HotKey="Ctrl+C"
IconSource="Copy"
IsEnabled="{OnPlatform Windows=True, Default=False}"
Text="Copy" />
</ui:FAMenuFlyout>
</controls:AdvancedImageBox.ContextFlyout>
</controls:AdvancedImageBox>
</DataTemplate>
</controls:DataTemplateSelector>
</ContentPresenter.ContentTemplate>
</ContentPresenter>
<!-- Info button -->
<Grid

48
StabilityMatrix.Core/Helper/ImageMetadata.cs

@ -3,7 +3,9 @@ using System.Text;
using System.Text.Json;
using ExifLibrary;
using MetadataExtractor;
using MetadataExtractor.Formats.Exif;
using MetadataExtractor.Formats.Png;
using MetadataExtractor.Formats.WebP;
using Microsoft.VisualBasic;
using StabilityMatrix.Core.Extensions;
using StabilityMatrix.Core.Models;
@ -79,6 +81,14 @@ public class ImageMetadata
string? ComfyNodes
) GetAllFileMetadata(FilePath filePath)
{
if (filePath.Extension.Equals(".webp", StringComparison.OrdinalIgnoreCase))
{
var paramsJson = ReadTextChunkFromWebp(filePath, ExifDirectoryBase.TagImageDescription);
var smProj = ReadTextChunkFromWebp(filePath, ExifDirectoryBase.TagSoftware);
return (null, paramsJson, smProj, null);
}
using var stream = filePath.Info.OpenRead();
using var reader = new BinaryReader(stream);
@ -234,32 +244,21 @@ public class ImageMetadata
return memoryStream;
}
public static string ReadTextChunkFromWebp(FilePath filePath, ExifTag exifTag)
/// <summary>
/// Reads an EXIF tag from a webp file and returns the value as string
/// </summary>
/// <param name="filePath">The webp file to read EXIF data from</param>
/// <param name="exifTag">Use <see cref="ExifDirectoryBase"/> constants for the tag you'd like to search for</param>
/// <returns></returns>
public static string ReadTextChunkFromWebp(FilePath filePath, int exifTag)
{
var sw = Stopwatch.StartNew();
try
{
var exifChunks = GetExifChunks(filePath);
if (exifChunks.Length == 0)
return string.Empty;
// write exifChunks to new memoryStream but skip first 6 bytes
using var newMemoryStream = new MemoryStream(exifChunks[6..]);
newMemoryStream.Seek(0, SeekOrigin.Begin);
var img = new MyTiffFile(newMemoryStream, Encoding.UTF8);
return img.Properties[exifTag]?.Value?.ToString() ?? string.Empty;
}
finally
{
sw.Stop();
Console.WriteLine($"ReadTextChunkFromWebp took {sw.ElapsedMilliseconds}ms");
}
var exifDirs = WebPMetadataReader.ReadMetadata(filePath).OfType<ExifIfd0Directory>().FirstOrDefault();
return exifDirs is null ? string.Empty : exifDirs.GetString(exifTag) ?? string.Empty;
}
public static IEnumerable<byte> AddMetadataToWebp(
byte[] inputImage,
GenerationParameters generationParameters
Dictionary<ExifTag, string> exifTagData
)
{
using var byteStream = new BinaryReader(new MemoryStream(inputImage));
@ -296,7 +295,11 @@ public class ImageMetadata
using var stream = new MemoryStream(exifBytes[6..]);
var img = new MyTiffFile(stream, Encoding.UTF8);
img.Properties.Set(ExifTag.ImageDescription, JsonSerializer.Serialize(generationParameters));
foreach (var (key, value) in exifTagData)
{
img.Properties.Set(key, value);
}
using var newStream = new MemoryStream();
img.Save(newStream);
@ -315,6 +318,7 @@ public class ImageMetadata
newImage = newImage.Concat(new byte[] { 0x00 }).ToArray();
}
// no clue why the minus 8 is needed but it is
var newImageSize = BitConverter.GetBytes(newImage.Length - 8);
newImage[4] = newImageSize[0];
newImage[5] = newImageSize[1];

44
StabilityMatrix.Core/Models/Database/LocalImageFile.cs

@ -1,4 +1,5 @@
using DynamicData.Tests;
using MetadataExtractor.Formats.Exif;
using StabilityMatrix.Core.Helper;
using StabilityMatrix.Core.Models.FileInterfaces;
using JsonSerializer = System.Text.Json.JsonSerializer;
@ -50,6 +51,17 @@ public record LocalImageFile
public (string? Parameters, string? ParametersJson, string? SMProject, string? ComfyNodes) ReadMetadata()
{
if (AbsolutePath.EndsWith("webp"))
{
var paramsJson = ImageMetadata.ReadTextChunkFromWebp(
AbsolutePath,
ExifDirectoryBase.TagImageDescription
);
var smProj = ImageMetadata.ReadTextChunkFromWebp(AbsolutePath, ExifDirectoryBase.TagSoftware);
return (null, paramsJson, smProj, null);
}
using var stream = new FileStream(AbsolutePath, FileMode.Open, FileAccess.Read, FileShare.Read);
using var reader = new BinaryReader(stream);
@ -71,6 +83,29 @@ public record LocalImageFile
// TODO: Support other types
const LocalImageFileType imageType = LocalImageFileType.Inference | LocalImageFileType.TextToImage;
if (filePath.Extension.Contains("webp"))
{
var paramsJson = ImageMetadata.ReadTextChunkFromWebp(
filePath,
ExifDirectoryBase.TagImageDescription
);
var parameters = string.IsNullOrWhiteSpace(paramsJson)
? null
: JsonSerializer.Deserialize<GenerationParameters>(paramsJson);
filePath.Info.Refresh();
return new LocalImageFile
{
AbsolutePath = filePath,
ImageType = imageType,
CreatedAt = filePath.Info.CreationTimeUtc,
LastModifiedAt = filePath.Info.LastWriteTimeUtc,
GenerationParameters = parameters,
ImageSize = new Size(parameters?.Width ?? 0, parameters?.Height ?? 0)
};
}
// Get metadata
using var stream = filePath.Info.OpenRead();
using var reader = new BinaryReader(stream);
@ -104,5 +139,12 @@ public record LocalImageFile
};
}
public static readonly HashSet<string> SupportedImageExtensions = [".png", ".jpg", ".jpeg", ".gif", ".webp"];
public static readonly HashSet<string> SupportedImageExtensions =
[
".png",
".jpg",
".jpeg",
".gif",
".webp"
];
}

9
StabilityMatrix.Core/Models/GenerationParameters.cs

@ -20,6 +20,15 @@ public partial record GenerationParameters
public int Width { get; set; }
public string? ModelHash { get; set; }
public string? ModelName { get; set; }
public int FrameCount { get; set; }
public int MotionBucketId { get; set; }
public int VideoQuality { get; set; }
public bool Lossless { get; set; }
public int Fps { get; set; }
public double OutputFps { get; set; }
public double MinCfg { get; set; }
public double AugmentationLevel { get; set; }
public string? VideoOutputMethod { get; set; }
public static bool TryParse(
string? text,

Loading…
Cancel
Save