Browse Source

Added image to video in inference (mostly)

pull/438/head
JT 12 months ago
parent
commit
e389085ec1
  1. 3
      StabilityMatrix.Avalonia/App.axaml
  2. 2
      StabilityMatrix.Avalonia/Controls/ModelCard.axaml
  3. 122
      StabilityMatrix.Avalonia/Controls/VideoGenerationSettingsCard.axaml
  4. 7
      StabilityMatrix.Avalonia/Controls/VideoGenerationSettingsCard.axaml.cs
  5. 86
      StabilityMatrix.Avalonia/Controls/VideoOutputSettingsCard.axaml
  6. 7
      StabilityMatrix.Avalonia/Controls/VideoOutputSettingsCard.axaml.cs
  7. 14
      StabilityMatrix.Avalonia/DesignData/DesignData.cs
  8. 8
      StabilityMatrix.Avalonia/Models/Inference/VideoOutputMethod.cs
  9. 1
      StabilityMatrix.Avalonia/Models/InferenceProjectDocument.cs
  10. 4
      StabilityMatrix.Avalonia/Models/InferenceProjectType.cs
  11. 12
      StabilityMatrix.Avalonia/StabilityMatrix.Avalonia.csproj
  12. 37
      StabilityMatrix.Avalonia/ViewModels/Base/InferenceGenerationViewModelBase.cs
  13. 243
      StabilityMatrix.Avalonia/ViewModels/Inference/InferenceImageToVideoViewModel.cs
  14. 5
      StabilityMatrix.Avalonia/ViewModels/Inference/ModelCardViewModel.cs
  15. 37
      StabilityMatrix.Avalonia/ViewModels/Inference/Video/ImgToVidModelCardViewModel.cs
  16. 90
      StabilityMatrix.Avalonia/ViewModels/Inference/Video/SvdImgToVidConditioningViewModel.cs
  17. 80
      StabilityMatrix.Avalonia/ViewModels/Inference/Video/VideoOutputSettingsCardViewModel.cs
  18. 221
      StabilityMatrix.Avalonia/Views/Inference/InferenceImageToVideoView.axaml
  19. 13
      StabilityMatrix.Avalonia/Views/Inference/InferenceImageToVideoView.axaml.cs
  20. 10
      StabilityMatrix.Avalonia/Views/InferencePage.axaml
  21. 7
      StabilityMatrix.Avalonia/Views/InferencePage.axaml.cs
  22. 2
      StabilityMatrix.Core/Models/Api/Comfy/NodeTypes/NodeConnections.cs
  23. 41
      StabilityMatrix.Core/Models/Api/Comfy/Nodes/ComfyNodeBuilder.cs

3
StabilityMatrix.Avalonia/App.axaml

@ -74,7 +74,8 @@
<StyleInclude Source="Controls/SelectableImageCard/SelectableImageButton.axaml"/>
<StyleInclude Source="Controls/SettingsAccountLinkExpander.axaml"/>
<StyleInclude Source="Controls/StarsRating.axaml"/>
<StyleInclude Source="Controls/VideoGenerationSettingsCard.axaml"/>
<StyleInclude Source="Controls/VideoOutputSettingsCard.axaml"/>
<Style Selector="DockControl">
<Setter Property="(DockProperties.ControlRecycling)" Value="{StaticResource ControlRecyclingKey}" />
</Style>

2
StabilityMatrix.Avalonia/Controls/ModelCard.axaml

@ -12,6 +12,7 @@
<Panel Width="400" Height="200">
<StackPanel Width="300" VerticalAlignment="Center">
<controls:ModelCard DataContext="{x:Static mocks:DesignData.ModelCardViewModel}" />
<controls:ModelCard DataContext="{x:Static mocks:DesignData.ImgToVidModelCardViewModel}" />
</StackPanel>
</Panel>
</Design.PreviewWith>
@ -135,6 +136,7 @@
<Button
Grid.Row="0"
Grid.Column="2"
IsVisible="{Binding !DisableSettings}"
Margin="8,0,0,0">
<ui:SymbolIcon FontSize="16" Symbol="Setting" />
<Button.Flyout>

122
StabilityMatrix.Avalonia/Controls/VideoGenerationSettingsCard.axaml

@ -0,0 +1,122 @@
<Styles xmlns="https://github.com/avaloniaui"
xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml"
xmlns:controls="using:StabilityMatrix.Avalonia.Controls"
x:DataType="video:SvdImgToVidConditioningViewModel"
xmlns:mocks="clr-namespace:StabilityMatrix.Avalonia.DesignData"
xmlns:video="clr-namespace:StabilityMatrix.Avalonia.ViewModels.Inference.Video"
xmlns:controls1="clr-namespace:FluentAvalonia.UI.Controls;assembly=FluentAvalonia"
xmlns:lang="clr-namespace:StabilityMatrix.Avalonia.Languages">
<Design.PreviewWith>
<Grid MinWidth="400">
<controls:VideoGenerationSettingsCard DataContext="{x:Static mocks:DesignData.SvdImgToVidConditioningViewModel}" />
</Grid>
</Design.PreviewWith>
<Style Selector="controls|VideoGenerationSettingsCard">
<!-- Set Defaults -->
<Setter Property="HorizontalAlignment" Value="Stretch" />
<Setter Property="Template">
<ControlTemplate>
<controls:Card Padding="8" HorizontalAlignment="{TemplateBinding HorizontalAlignment}">
<Grid Margin="4" RowDefinitions="Auto,Auto,Auto,Auto,Auto" ColumnDefinitions="Auto, *">
<TextBlock
Grid.Row="0"
Grid.Column="0"
Margin="0,0,8,0"
VerticalAlignment="Center"
Text="Frames" />
<controls1:NumberBox
Grid.Row="0"
Grid.Column="1"
SelectionHighlightColor="Transparent"
Value="{Binding NumFrames}"
Margin="8,0,0,0"
SimpleNumberFormat="F0"
SmallChange="1"
HorizontalAlignment="Stretch"
SpinButtonPlacementMode="Inline"/>
<TextBlock
Grid.Row="1"
Grid.Column="0"
Margin="0,8,8,0"
VerticalAlignment="Center"
Text="Frames Per Second" />
<controls1:NumberBox
Grid.Row="1"
Grid.Column="1"
SelectionHighlightColor="Transparent"
Value="{Binding Fps}"
Margin="8,8,0,0"
SimpleNumberFormat="F0"
SmallChange="1"
HorizontalAlignment="Stretch"
SpinButtonPlacementMode="Inline"/>
<TextBlock
Grid.Row="2"
Grid.Column="0"
Margin="0,8,8,0"
VerticalAlignment="Center"
Text="Min CFG" />
<controls1:NumberBox
Margin="8,8,0,0"
Grid.Row="2"
Grid.Column="1"
SelectionHighlightColor="Transparent"
Value="{Binding MinCfg}"
SimpleNumberFormat="F0"
SmallChange="1"
HorizontalAlignment="Stretch"
SpinButtonPlacementMode="Inline"/>
<TextBlock
Margin="0,8,8,0"
Grid.Row="3"
Grid.Column="0"
VerticalAlignment="Center"
Text="Motion Bucket ID" />
<controls1:NumberBox
Margin="8,8,0,0"
Grid.Row="3"
Grid.Column="1"
SelectionHighlightColor="Transparent"
Value="{Binding MotionBucketId}"
SimpleNumberFormat="F0"
SmallChange="1"
HorizontalAlignment="Stretch"
SpinButtonPlacementMode="Inline"/>
<StackPanel Grid.Column="0"
Grid.ColumnSpan="2"
Grid.Row="4"
Margin="0,16,0,0">
<Grid ColumnDefinitions="*,Auto">
<TextBlock
VerticalAlignment="Center"
Text="Augmentation Level"/>
<controls1:NumberBox
Grid.Column="1"
Margin="4,0,0,0"
ValidationMode="InvalidInputOverwritten"
SmallChange="0.01"
SimpleNumberFormat="F2"
Value="{Binding AugmentationLevel}"
HorizontalAlignment="Stretch"
MinWidth="100"
SpinButtonPlacementMode="Compact"/>
</Grid>
<Slider
Minimum="0"
Maximum="10"
Value="{Binding AugmentationLevel}"
TickFrequency="1"
Margin="0,0,0,-4"
TickPlacement="BottomRight"/>
</StackPanel>
</Grid>
</controls:Card>
</ControlTemplate>
</Setter>
</Style>
</Styles>

7
StabilityMatrix.Avalonia/Controls/VideoGenerationSettingsCard.axaml.cs

@ -0,0 +1,7 @@
using Avalonia.Controls.Primitives;
using StabilityMatrix.Core.Attributes;
namespace StabilityMatrix.Avalonia.Controls;
[Transient]
public class VideoGenerationSettingsCard : TemplatedControl { }

86
StabilityMatrix.Avalonia/Controls/VideoOutputSettingsCard.axaml

@ -0,0 +1,86 @@
<Styles xmlns="https://github.com/avaloniaui"
xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml"
xmlns:controls="using:StabilityMatrix.Avalonia.Controls"
x:DataType="video:VideoOutputSettingsCardViewModel"
xmlns:mocks="clr-namespace:StabilityMatrix.Avalonia.DesignData"
xmlns:video="clr-namespace:StabilityMatrix.Avalonia.ViewModels.Inference.Video"
xmlns:controls1="clr-namespace:FluentAvalonia.UI.Controls;assembly=FluentAvalonia"
xmlns:lang="clr-namespace:StabilityMatrix.Avalonia.Languages">
<Design.PreviewWith>
<Grid MinWidth="400">
<controls:VideoOutputSettingsCard DataContext="{x:Static mocks:DesignData.SvdImgToVidConditioningViewModel}" />
</Grid>
</Design.PreviewWith>
<Style Selector="controls|VideoOutputSettingsCard">
<!-- Set Defaults -->
<Setter Property="HorizontalAlignment" Value="Stretch" />
<Setter Property="Template">
<ControlTemplate>
<controls:Card Padding="8" HorizontalAlignment="{TemplateBinding HorizontalAlignment}">
<Grid Margin="4" RowDefinitions="Auto,Auto,Auto,Auto,Auto" ColumnDefinitions="Auto, *">
<TextBlock
Grid.Row="0"
Grid.Column="0"
Margin="0,0,8,0"
VerticalAlignment="Center"
Text="Frames Per Second" />
<controls1:NumberBox
Grid.Row="0"
Grid.Column="1"
SelectionHighlightColor="Transparent"
Value="{Binding Fps}"
Margin="8,0,0,0"
SimpleNumberFormat="F0"
SmallChange="1"
HorizontalAlignment="Stretch"
SpinButtonPlacementMode="Inline"/>
<TextBlock
Grid.Row="1"
Grid.Column="0"
Margin="0,8,8,0"
VerticalAlignment="Center"
Text="Lossless" />
<CheckBox
Grid.Row="1"
Grid.Column="1"
IsChecked="{Binding Lossless}"
Margin="8,8,0,0"
HorizontalAlignment="Stretch"/>
<TextBlock
Grid.Row="2"
Grid.Column="0"
Margin="0,8,8,0"
VerticalAlignment="Center"
Text="Quality" />
<controls1:NumberBox
Margin="8,8,0,0"
Grid.Row="2"
Grid.Column="1"
SelectionHighlightColor="Transparent"
Value="{Binding Quality}"
SimpleNumberFormat="F0"
SmallChange="1"
Maximum="100"
HorizontalAlignment="Stretch"
SpinButtonPlacementMode="Inline"/>
<TextBlock
Margin="0,8,8,0"
Grid.Row="3"
Grid.Column="0"
VerticalAlignment="Center"
Text="Method" />
<ComboBox Grid.Row="3" Grid.Column="1"
Margin="8,8,0,0"
MinWidth="100"
ItemsSource="{Binding AvailableMethods}"
SelectedIndex="{Binding SelectedMethod}"/>
</Grid>
</controls:Card>
</ControlTemplate>
</Setter>
</Style>
</Styles>

7
StabilityMatrix.Avalonia/Controls/VideoOutputSettingsCard.axaml.cs

@ -0,0 +1,7 @@
using Avalonia.Controls.Primitives;
using StabilityMatrix.Core.Attributes;
namespace StabilityMatrix.Avalonia.Controls;
[Transient]
public class VideoOutputSettingsCard : TemplatedControl { }

14
StabilityMatrix.Avalonia/DesignData/DesignData.cs

@ -24,6 +24,7 @@ using StabilityMatrix.Avalonia.ViewModels.Dialogs;
using StabilityMatrix.Avalonia.ViewModels.Progress;
using StabilityMatrix.Avalonia.ViewModels.Inference;
using StabilityMatrix.Avalonia.ViewModels.Inference.Modules;
using StabilityMatrix.Avalonia.ViewModels.Inference.Video;
using StabilityMatrix.Avalonia.ViewModels.OutputsPage;
using StabilityMatrix.Avalonia.ViewModels.Settings;
using StabilityMatrix.Core.Api;
@ -644,6 +645,14 @@ The gallery images are often inpainted, but you will get something very similar
vm.OutputProgress.Maximum = 30;
vm.OutputProgress.Text = "Sampler 10/30";
});
public static InferenceImageToVideoViewModel InferenceImageToVideoViewModel =>
DialogFactory.Get<InferenceImageToVideoViewModel>(vm =>
{
vm.OutputProgress.Value = 10;
vm.OutputProgress.Maximum = 30;
vm.OutputProgress.Text = "Sampler 10/30";
});
public static InferenceImageUpscaleViewModel InferenceImageUpscaleViewModel =>
DialogFactory.Get<InferenceImageUpscaleViewModel>();
@ -655,6 +664,8 @@ The gallery images are often inpainted, but you will get something very similar
new() { State = ProgressState.Success };
public static SeedCardViewModel SeedCardViewModel => new();
public static SvdImgToVidConditioningViewModel SvdImgToVidConditioningViewModel => new();
public static VideoOutputSettingsCardViewModel VideoOutputSettingsCardViewModel => new();
public static SamplerCardViewModel SamplerCardViewModel =>
DialogFactory.Get<SamplerCardViewModel>(vm =>
@ -684,6 +695,9 @@ The gallery images are often inpainted, but you will get something very similar
public static ModelCardViewModel ModelCardViewModel => DialogFactory.Get<ModelCardViewModel>();
public static ImgToVidModelCardViewModel ImgToVidModelCardViewModel =>
DialogFactory.Get<ImgToVidModelCardViewModel>();
public static ImageGalleryCardViewModel ImageGalleryCardViewModel =>
DialogFactory.Get<ImageGalleryCardViewModel>(vm =>
{

8
StabilityMatrix.Avalonia/Models/Inference/VideoOutputMethod.cs

@ -0,0 +1,8 @@
namespace StabilityMatrix.Avalonia.Models.Inference;
public enum VideoOutputMethod
{
Fastest,
Default,
Slowest,
}

1
StabilityMatrix.Avalonia/Models/InferenceProjectDocument.cs

@ -32,6 +32,7 @@ public class InferenceProjectDocument : ICloneable
{
InferenceTextToImageViewModel => InferenceProjectType.TextToImage,
InferenceImageUpscaleViewModel => InferenceProjectType.Upscale,
InferenceImageToVideoViewModel => InferenceProjectType.ImageToVideo,
_
=> throw new InvalidOperationException(
$"Unknown loadable model type: {loadableModel.GetType()}"

4
StabilityMatrix.Avalonia/Models/InferenceProjectType.cs

@ -9,7 +9,8 @@ public enum InferenceProjectType
TextToImage,
ImageToImage,
Inpainting,
Upscale
Upscale,
ImageToVideo
}
public static class InferenceProjectTypeExtensions
@ -22,6 +23,7 @@ public static class InferenceProjectTypeExtensions
InferenceProjectType.ImageToImage => null,
InferenceProjectType.Inpainting => null,
InferenceProjectType.Upscale => typeof(InferenceImageUpscaleViewModel),
InferenceProjectType.ImageToVideo => typeof(InferenceImageToVideoViewModel),
InferenceProjectType.Unknown => null,
_ => throw new ArgumentOutOfRangeException(nameof(type), type, null)
};

12
StabilityMatrix.Avalonia/StabilityMatrix.Avalonia.csproj

@ -138,6 +138,18 @@
<AutoGen>True</AutoGen>
<DependentUpon>Resources.resx</DependentUpon>
</Compile>
<Compile Update="Views\Inference\InferenceImageToVideoView.axaml.cs">
<DependentUpon>InferenceImageToVideoView.axaml</DependentUpon>
<SubType>Code</SubType>
</Compile>
<Compile Update="Controls\VideoGenerationSettingsCard.axaml.cs">
<DependentUpon>VideoGenerationSettingsCard.axaml</DependentUpon>
<SubType>Code</SubType>
</Compile>
<Compile Update="Controls\VideoOutputSettingsCard.axaml.cs">
<DependentUpon>VideoOutputSettingsCard.axaml</DependentUpon>
<SubType>Code</SubType>
</Compile>
</ItemGroup>
<!-- set HUSKY to 0 to disable, or opt-in during CI by setting HUSKY to 1 -->

37
StabilityMatrix.Avalonia/ViewModels/Base/InferenceGenerationViewModelBase.cs

@ -418,19 +418,34 @@ public abstract partial class InferenceGenerationViewModelBase
);
}
var bytesWithMetadata = PngDataHelper.AddMetadata(imageArray, parameters, project);
// Write using generated name
var filePath = await WriteOutputImageAsync(
new MemoryStream(bytesWithMetadata),
args,
i + 1,
images.Count
);
if (comfyImage.FileName.EndsWith(".png"))
{
var bytesWithMetadata = PngDataHelper.AddMetadata(imageArray, parameters, project);
// Write using generated name
var filePath = await WriteOutputImageAsync(
new MemoryStream(bytesWithMetadata),
args,
i + 1,
images.Count
);
outputImages.Add(new ImageSource(filePath));
outputImages.Add(new ImageSource(filePath));
EventManager.Instance.OnImageFileAdded(filePath);
}
else
{
// Write using generated name
var filePath = await WriteOutputImageAsync(
new MemoryStream(imageArray),
args,
i + 1,
images.Count
);
EventManager.Instance.OnImageFileAdded(filePath);
outputImages.Add(new ImageSource(filePath));
EventManager.Instance.OnImageFileAdded(filePath);
}
}
// Download all images to make grid, if multiple

243
StabilityMatrix.Avalonia/ViewModels/Inference/InferenceImageToVideoViewModel.cs

@ -0,0 +1,243 @@
using System;
using System.Collections.Generic;
using System.ComponentModel.DataAnnotations;
using System.Drawing;
using System.Linq;
using System.Text.Json.Nodes;
using System.Text.Json.Serialization;
using System.Threading;
using System.Threading.Tasks;
using NLog;
using StabilityMatrix.Avalonia.Extensions;
using StabilityMatrix.Avalonia.Models;
using StabilityMatrix.Avalonia.Models.Inference;
using StabilityMatrix.Avalonia.Services;
using StabilityMatrix.Avalonia.ViewModels.Base;
using StabilityMatrix.Avalonia.ViewModels.Inference.Modules;
using StabilityMatrix.Avalonia.ViewModels.Inference.Video;
using StabilityMatrix.Avalonia.Views.Inference;
using StabilityMatrix.Core.Attributes;
using StabilityMatrix.Core.Models;
using StabilityMatrix.Core.Models.Api.Comfy.Nodes;
using StabilityMatrix.Core.Services;
#pragma warning disable CS0657 // Not a valid attribute location for this declaration
namespace StabilityMatrix.Avalonia.ViewModels.Inference;
[View(typeof(InferenceImageToVideoView), persistent: true)]
[ManagedService]
[Transient]
public class InferenceImageToVideoViewModel
: InferenceGenerationViewModelBase,
IParametersLoadableState
{
private static readonly Logger Logger = LogManager.GetCurrentClassLogger();
private readonly INotificationService notificationService;
private readonly IModelIndexService modelIndexService;
[JsonIgnore]
public StackCardViewModel StackCardViewModel { get; }
[JsonPropertyName("Model")]
public ImgToVidModelCardViewModel ModelCardViewModel { get; }
[JsonPropertyName("Sampler")]
public SamplerCardViewModel SamplerCardViewModel { get; }
[JsonPropertyName("BatchSize")]
public BatchSizeCardViewModel BatchSizeCardViewModel { get; }
[JsonPropertyName("Seed")]
public SeedCardViewModel SeedCardViewModel { get; }
[JsonPropertyName("ImageLoader")]
public SelectImageCardViewModel SelectImageCardViewModel { get; }
[JsonPropertyName("Conditioning")]
public SvdImgToVidConditioningViewModel SvdImgToVidConditioningViewModel { get; }
[JsonPropertyName("VideoOutput")]
public VideoOutputSettingsCardViewModel VideoOutputSettingsCardViewModel { get; }
public InferenceImageToVideoViewModel(
INotificationService notificationService,
IInferenceClientManager inferenceClientManager,
ISettingsManager settingsManager,
ServiceManager<ViewModelBase> vmFactory,
IModelIndexService modelIndexService
)
: base(vmFactory, inferenceClientManager, notificationService, settingsManager)
{
this.notificationService = notificationService;
this.modelIndexService = modelIndexService;
// Get sub view models from service manager
SeedCardViewModel = vmFactory.Get<SeedCardViewModel>();
SeedCardViewModel.GenerateNewSeed();
ModelCardViewModel = vmFactory.Get<ImgToVidModelCardViewModel>();
SamplerCardViewModel = vmFactory.Get<SamplerCardViewModel>(samplerCard =>
{
samplerCard.IsDimensionsEnabled = true;
samplerCard.IsCfgScaleEnabled = true;
samplerCard.IsSamplerSelectionEnabled = true;
samplerCard.IsSchedulerSelectionEnabled = true;
});
BatchSizeCardViewModel = vmFactory.Get<BatchSizeCardViewModel>();
SelectImageCardViewModel = vmFactory.Get<SelectImageCardViewModel>();
SvdImgToVidConditioningViewModel = vmFactory.Get<SvdImgToVidConditioningViewModel>();
VideoOutputSettingsCardViewModel = vmFactory.Get<VideoOutputSettingsCardViewModel>();
StackCardViewModel = vmFactory.Get<StackCardViewModel>();
StackCardViewModel.AddCards(
ModelCardViewModel,
SvdImgToVidConditioningViewModel,
SamplerCardViewModel,
SeedCardViewModel,
VideoOutputSettingsCardViewModel,
BatchSizeCardViewModel
);
}
/// <inheritdoc />
protected override void BuildPrompt(BuildPromptEventArgs args)
{
base.BuildPrompt(args);
var builder = args.Builder;
builder.Connections.Seed = args.SeedOverride switch
{
{ } seed => Convert.ToUInt64(seed),
_ => Convert.ToUInt64(SeedCardViewModel.Seed)
};
// Load models
ModelCardViewModel.ApplyStep(args);
// Setup latent from image
var imageLoad = builder.Nodes.AddTypedNode(
new ComfyNodeBuilder.LoadImage
{
Name = builder.Nodes.GetUniqueName("ControlNet_LoadImage"),
Image =
SelectImageCardViewModel.ImageSource?.GetHashGuidFileNameCached("Inference")
?? throw new ValidationException()
}
);
builder.Connections.Primary = imageLoad.Output1;
builder.Connections.PrimarySize =
SelectImageCardViewModel.CurrentBitmapSize
?? new Size(SamplerCardViewModel.Width, SamplerCardViewModel.Height);
// Setup img2vid stuff
// Set width & height from SamplerCard
SvdImgToVidConditioningViewModel.Width = SamplerCardViewModel.Width;
SvdImgToVidConditioningViewModel.Height = SamplerCardViewModel.Height;
SvdImgToVidConditioningViewModel.ApplyStep(args);
// Setup Sampler and Refiner if enabled
SamplerCardViewModel.ApplyStep(args);
// Animated webp output
VideoOutputSettingsCardViewModel.ApplyStep(args);
}
/// <inheritdoc />
protected override IEnumerable<ImageSource> GetInputImages()
{
if (SelectImageCardViewModel.ImageSource is { } image)
{
yield return image;
}
}
/// <inheritdoc />
protected override async Task GenerateImageImpl(
GenerateOverrides overrides,
CancellationToken cancellationToken
)
{
if (!await CheckClientConnectedWithPrompt() || !ClientManager.IsConnected)
{
return;
}
// If enabled, randomize the seed
var seedCard = StackCardViewModel.GetCard<SeedCardViewModel>();
if (overrides is not { UseCurrentSeed: true } && seedCard.IsRandomizeEnabled)
{
seedCard.GenerateNewSeed();
}
var batches = BatchSizeCardViewModel.BatchCount;
var batchArgs = new List<ImageGenerationEventArgs>();
for (var i = 0; i < batches; i++)
{
var seed = seedCard.Seed + i;
var buildPromptArgs = new BuildPromptEventArgs
{
Overrides = overrides,
SeedOverride = seed
};
BuildPrompt(buildPromptArgs);
var generationArgs = new ImageGenerationEventArgs
{
Client = ClientManager.Client,
Nodes = buildPromptArgs.Builder.ToNodeDictionary(),
OutputNodeNames = buildPromptArgs.Builder.Connections.OutputNodeNames.ToArray(),
Parameters = SaveStateToParameters(new GenerationParameters()),
Project = InferenceProjectDocument.FromLoadable(this),
// Only clear output images on the first batch
ClearOutputImages = i == 0
};
batchArgs.Add(generationArgs);
}
// Run batches
foreach (var args in batchArgs)
{
await RunGeneration(args, cancellationToken);
}
}
/// <inheritdoc />
public void LoadStateFromParameters(GenerationParameters parameters)
{
SamplerCardViewModel.LoadStateFromParameters(parameters);
ModelCardViewModel.LoadStateFromParameters(parameters);
SeedCardViewModel.Seed = Convert.ToInt64(parameters.Seed);
}
/// <inheritdoc />
public GenerationParameters SaveStateToParameters(GenerationParameters parameters)
{
parameters = SamplerCardViewModel.SaveStateToParameters(parameters);
parameters = ModelCardViewModel.SaveStateToParameters(parameters);
parameters.Seed = (ulong)SeedCardViewModel.Seed;
return parameters;
}
// Migration for v2 deserialization
public override void LoadStateFromJsonObject(JsonObject state, int version)
{
if (version > 2)
{
LoadStateFromJsonObject(state);
}
}
}

5
StabilityMatrix.Avalonia/ViewModels/Inference/ModelCardViewModel.cs

@ -38,6 +38,9 @@ public partial class ModelCardViewModel
[ObservableProperty]
private bool isVaeSelectionEnabled;
[ObservableProperty]
private bool disableSettings;
public string? SelectedModelName => SelectedModel?.RelativePath;
public string? SelectedVaeName => SelectedVae?.RelativePath;
@ -50,7 +53,7 @@ public partial class ModelCardViewModel
}
/// <inheritdoc />
public void ApplyStep(ModuleApplyStepEventArgs e)
public virtual void ApplyStep(ModuleApplyStepEventArgs e)
{
// Load base checkpoint
var baseLoader = e.Nodes.AddTypedNode(

37
StabilityMatrix.Avalonia/ViewModels/Inference/Video/ImgToVidModelCardViewModel.cs

@ -0,0 +1,37 @@
using System.ComponentModel.DataAnnotations;
using StabilityMatrix.Avalonia.Controls;
using StabilityMatrix.Avalonia.Models.Inference;
using StabilityMatrix.Avalonia.Services;
using StabilityMatrix.Core.Attributes;
using StabilityMatrix.Core.Models.Api.Comfy.Nodes;
namespace StabilityMatrix.Avalonia.ViewModels.Inference.Video;
[View(typeof(ModelCard))]
[ManagedService]
[Transient]
public class ImgToVidModelCardViewModel : ModelCardViewModel
{
public ImgToVidModelCardViewModel(IInferenceClientManager clientManager)
: base(clientManager)
{
DisableSettings = true;
}
public override void ApplyStep(ModuleApplyStepEventArgs e)
{
var imgToVidLoader = e.Nodes.AddTypedNode(
new ComfyNodeBuilder.ImageOnlyCheckpointLoader
{
Name = "ImageOnlyCheckpointLoader",
CkptName =
SelectedModel?.RelativePath
?? throw new ValidationException("Model not selected")
}
);
e.Builder.Connections.BaseModel = imgToVidLoader.Output1;
e.Builder.Connections.BaseClipVision = imgToVidLoader.Output2;
e.Builder.Connections.BaseVAE = imgToVidLoader.Output3;
}
}

90
StabilityMatrix.Avalonia/ViewModels/Inference/Video/SvdImgToVidConditioningViewModel.cs

@ -0,0 +1,90 @@
using System.ComponentModel.DataAnnotations;
using CommunityToolkit.Mvvm.ComponentModel;
using StabilityMatrix.Avalonia.Controls;
using StabilityMatrix.Avalonia.Models;
using StabilityMatrix.Avalonia.Models.Inference;
using StabilityMatrix.Avalonia.ViewModels.Base;
using StabilityMatrix.Core.Attributes;
using StabilityMatrix.Core.Models;
using StabilityMatrix.Core.Models.Api.Comfy.Nodes;
namespace StabilityMatrix.Avalonia.ViewModels.Inference.Video;
[View(typeof(VideoGenerationSettingsCard))]
[ManagedService]
[Transient]
public partial class SvdImgToVidConditioningViewModel
: LoadableViewModelBase,
IParametersLoadableState,
IComfyStep
{
[ObservableProperty]
private int width = 1024;
[ObservableProperty]
private int height = 576;
[ObservableProperty]
private int numFrames = 14;
[ObservableProperty]
private int motionBucketId = 127;
[ObservableProperty]
private int fps = 6;
[ObservableProperty]
private double augmentationLevel;
[ObservableProperty]
private double minCfg = 1.0d;
public void LoadStateFromParameters(GenerationParameters parameters)
{
// TODO
}
public GenerationParameters SaveStateToParameters(GenerationParameters parameters)
{
// TODO
return parameters;
}
public void ApplyStep(ModuleApplyStepEventArgs e)
{
// do VideoLinearCFGGuidance stuff first
var cfgGuidanceNode = e.Nodes.AddTypedNode(
new ComfyNodeBuilder.VideoLinearCFGGuidance
{
Name = e.Nodes.GetUniqueName("LinearCfgGuidance"),
Model =
e.Builder.Connections.BaseModel
?? throw new ValidationException("Model not selected"),
MinCfg = MinCfg
}
);
e.Builder.Connections.BaseModel = cfgGuidanceNode.Output;
// then do the SVD stuff
var svdImgToVidConditioningNode = e.Nodes.AddTypedNode(
new ComfyNodeBuilder.SVD_img2vid_Conditioning
{
ClipVision = e.Builder.Connections.BaseClipVision!,
InitImage = e.Builder.GetPrimaryAsImage(),
Vae = e.Builder.Connections.BaseVAE!,
Name = e.Nodes.GetUniqueName("SvdImgToVidConditioning"),
Width = Width,
Height = Height,
VideoFrames = NumFrames,
MotionBucketId = MotionBucketId,
Fps = Fps,
AugmentationLevel = AugmentationLevel
}
);
e.Builder.Connections.BaseConditioning = svdImgToVidConditioningNode.Output1;
e.Builder.Connections.BaseNegativeConditioning = svdImgToVidConditioningNode.Output2;
e.Builder.Connections.Primary = svdImgToVidConditioningNode.Output3;
}
}

80
StabilityMatrix.Avalonia/ViewModels/Inference/Video/VideoOutputSettingsCardViewModel.cs

@ -0,0 +1,80 @@
using System;
using System.Collections.Generic;
using System.Linq;
using CommunityToolkit.Mvvm.ComponentModel;
using StabilityMatrix.Avalonia.Controls;
using StabilityMatrix.Avalonia.Models;
using StabilityMatrix.Avalonia.Models.Inference;
using StabilityMatrix.Avalonia.ViewModels.Base;
using StabilityMatrix.Core.Attributes;
using StabilityMatrix.Core.Models;
using StabilityMatrix.Core.Models.Api.Comfy.Nodes;
namespace StabilityMatrix.Avalonia.ViewModels.Inference.Video;
[View(typeof(VideoOutputSettingsCard))]
[ManagedService]
[Transient]
public partial class VideoOutputSettingsCardViewModel
: LoadableViewModelBase,
IParametersLoadableState,
IComfyStep
{
[ObservableProperty]
private double fps = 6;
[ObservableProperty]
private bool lossless = true;
[ObservableProperty]
private int quality = 85;
[ObservableProperty]
private VideoOutputMethod selectedMethod = VideoOutputMethod.Default;
[ObservableProperty]
private List<VideoOutputMethod> availableMethods = Enum.GetValues<VideoOutputMethod>().ToList();
public void LoadStateFromParameters(GenerationParameters parameters)
{
// TODO
}
public GenerationParameters SaveStateToParameters(GenerationParameters parameters)
{
// TODO
return parameters;
}
public void ApplyStep(ModuleApplyStepEventArgs e)
{
if (e.Builder.Connections.Primary is null)
throw new ArgumentException("No Primary");
var image = e.Builder.Connections.Primary.Match(
_ =>
e.Builder.GetPrimaryAsImage(
e.Builder.Connections.PrimaryVAE
?? e.Builder.Connections.RefinerVAE
?? e.Builder.Connections.BaseVAE
?? throw new ArgumentException("No Primary, Refiner, or Base VAE")
),
image => image
);
var outputStep = e.Nodes.AddTypedNode(
new ComfyNodeBuilder.SaveAnimatedWEBP
{
Name = e.Nodes.GetUniqueName("SaveAnimatedWEBP"),
Images = image,
FilenamePrefix = "InferenceVideo",
Fps = Fps,
Lossless = Lossless,
Quality = Quality,
Method = SelectedMethod.ToString().ToLowerInvariant()
}
);
e.Builder.Connections.OutputNodes.Add(outputStep);
}
}

221
StabilityMatrix.Avalonia/Views/Inference/InferenceImageToVideoView.axaml

@ -0,0 +1,221 @@
<dock:DockUserControlBase
x:Class="StabilityMatrix.Avalonia.Views.Inference.InferenceImageToVideoView"
xmlns="https://github.com/avaloniaui"
xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml"
xmlns:controls="clr-namespace:StabilityMatrix.Avalonia.Controls"
xmlns:d="http://schemas.microsoft.com/expression/blend/2008"
xmlns:icons="clr-namespace:Projektanker.Icons.Avalonia;assembly=Projektanker.Icons.Avalonia"
xmlns:mc="http://schemas.openxmlformats.org/markup-compatibility/2006"
xmlns:mocks="clr-namespace:StabilityMatrix.Avalonia.DesignData"
xmlns:vmInference="using:StabilityMatrix.Avalonia.ViewModels.Inference"
xmlns:dock="clr-namespace:StabilityMatrix.Avalonia.Controls.Dock"
xmlns:modelsInference="clr-namespace:StabilityMatrix.Avalonia.Models.Inference"
d:DataContext="{x:Static mocks:DesignData.InferenceImageToVideoViewModel}"
d:DesignHeight="800"
d:DesignWidth="1000"
x:DataType="vmInference:InferenceImageToVideoViewModel"
Focusable="True"
mc:Ignorable="d">
<Grid Margin="2,0,2,8">
<DockControl
x:Name="Dock"
InitializeFactory="True"
InitializeLayout="True">
<DockControl.Factory>
<Factory />
</DockControl.Factory>
<RootDock
x:Name="Root"
DefaultDockable="{Binding #MainLayout}"
Id="Root"
IsCollapsable="False">
<ProportionalDock
x:Name="MainLayout"
Id="MainLayout"
Orientation="Horizontal">
<!-- Left Pane -->
<ProportionalDock
Proportion="0.3"
Orientation="Vertical">
<ToolDock
x:Name="SelectImagePane"
Alignment="Left"
Id="ConfigPane"
Proportion="0.3">
<Tool
x:Name="SelectImageTool"
Title="Image Input"
x:DataType="Tool"
Id="ConfigTool"
CanClose="False">
<controls:SelectImageCard
DataContext="{Binding #Dock.((vmInference:InferenceImageToVideoViewModel)DataContext).SelectImageCardViewModel}" />
</Tool>
</ToolDock>
<ToolDock
x:Name="ConfigPane"
Alignment="Left"
Id="ConfigPane"
Proportion="0.6">
<Tool
x:Name="ConfigTool"
Title="Config"
x:DataType="Tool"
Id="ConfigTool"
CanClose="False">
<controls:StackCard
Opacity="1"
DataContext="{ReflectionBinding ElementName=Dock, Path=DataContext.StackCardViewModel}" />
</Tool>
</ToolDock>
</ProportionalDock>
<ProportionalDockSplitter x:Name="Splitter1" Id="Splitter1" />
<!-- Middle Pane -->
<ProportionalDock
Proportion="0.5"
Orientation="Vertical">
<ToolDock
x:Name="ImageGalleryPane"
Alignment="Right"
Id="ImageGalleryPane"
Proportion="0.9">
<Tool
x:Name="ImageGalleryTool"
Title="Image Output"
x:DataType="Tool"
Id="ImageGalleryTool"
CanClose="False">
<Grid
x:CompileBindings="False"
DataContext="{Binding ElementName=Dock, Path=DataContext}">
<controls:ImageGalleryCard
Grid.Row="0"
DataContext="{Binding ImageGalleryCardViewModel}" />
<StackPanel
DataContext="{Binding OutputProgress}"
Margin="2,1,2,4"
Spacing="4"
VerticalAlignment="Top">
<ProgressBar
IsVisible="{Binding IsProgressVisible}"
IsIndeterminate="{Binding IsIndeterminate}"
Maximum="{Binding Maximum}"
Value="{Binding Value}" />
<TextBlock
Margin="0,10,0,0"
IsVisible="{Binding IsTextVisible}"
TextAlignment="Center"
Text="{Binding Text}" />
</StackPanel>
</Grid>
</Tool>
</ToolDock>
<ToolDock x:Name="GeneratePane"
Alignment="Top"
Id="ConfigPane"
Proportion="0.1">
<Tool
x:Name="GenerateTool"
Title="Generate"
x:DataType="Tool"
Id="GenerateTool"
CanClose="False">
<!-- Generate Buttons -->
<StackPanel Margin="0,6,0,0"
x:CompileBindings="False"
DataContext="{Binding ElementName=Dock, Path=DataContext}">
<controls:Card Padding="8">
<Grid
HorizontalAlignment="Center"
ColumnDefinitions="Auto,*,Auto"
RowDefinitions="Auto,*">
<Grid.Styles>
<Style Selector="Button">
<Setter Property="MinHeight" Value="32" />
</Style>
</Grid.Styles>
<!-- Main Generate Button -->
<Button
x:Name="GenerateButton"
Grid.Row="1"
Grid.Column="1"
Width="130"
HorizontalAlignment="Stretch"
Classes="accent"
Command="{Binding GenerateImageCommand}"
CommandParameter="{x:Static modelsInference:GenerateFlags.None}"
IsVisible="{Binding !GenerateImageCommand.CanBeCanceled}">
<Panel>
<TextBlock IsVisible="{Binding #GenerateButton.IsEnabled}"
Text="Generate Image" />
<controls:ProgressRing IsIndeterminate="True"
IsVisible="{Binding !#GenerateButton.IsEnabled}" />
</Panel>
</Button>
<!-- Cancel Button -->
<Button
x:Name="CancelButton"
Grid.Row="1"
Grid.Column="1"
Width="130"
HorizontalAlignment="Stretch"
icons:Attached.Icon="fa-solid fa-stop"
Command="{Binding GenerateImageCancelCommand}"
IsVisible="{Binding GenerateImageCommand.CanBeCanceled}" />
<Button
Grid.Row="1"
Grid.Column="2"
Margin="4,0"
Padding="12,8"
HorizontalAlignment="Left"
icons:Attached.Icon="fa-solid fa-repeat"
Classes="transparent"
Command="{Binding GenerateImageCommand}"
CommandParameter="{x:Static modelsInference:GenerateFlags.UseCurrentSeed}"
ToolTip.Tip="Generate with current seed" />
</Grid>
</controls:Card>
</StackPanel>
</Tool>
</ToolDock>
</ProportionalDock>
<ProportionalDockSplitter x:Name="Splitter3" Id="Splitter3" />
<!-- Right Pane -->
<ToolDock
x:Name="ImageFolderPane"
Alignment="Right"
Id="ImageFolderPane"
Proportion="0.2">
<Tool
x:Name="ImageFolderTool"
x:DataType="Tool"
Title="Gallery"
Id="ImageFolderTool"
CanClose="False">
<Grid
x:CompileBindings="False"
DataContext="{Binding ElementName=Dock, Path=DataContext}">
<controls:ImageFolderCard
DataContext="{Binding ImageFolderCardViewModel}" />
</Grid>
</Tool>
</ToolDock>
</ProportionalDock>
</RootDock>
</DockControl>
</Grid>
</dock:DockUserControlBase>

13
StabilityMatrix.Avalonia/Views/Inference/InferenceImageToVideoView.axaml.cs

@ -0,0 +1,13 @@
using StabilityMatrix.Avalonia.Controls.Dock;
using StabilityMatrix.Core.Attributes;
namespace StabilityMatrix.Avalonia.Views.Inference;
[Transient]
public partial class InferenceImageToVideoView : DockUserControlBase
{
public InferenceImageToVideoView()
{
InitializeComponent();
}
}

10
StabilityMatrix.Avalonia/Views/InferencePage.axaml

@ -68,6 +68,16 @@
Symbol="ResizeImage"/>
</ui:CommandBarButton.IconSource>
</ui:CommandBarButton>
<ui:CommandBarButton
Click="AddTabMenu_ImgToVideo_OnClick"
Label="Image to Video"
ToolTip.Tip="Image to Video">
<ui:CommandBarButton.IconSource>
<fluentIcons:SymbolIconSource
FontSize="10"
Symbol="Video"/>
</ui:CommandBarButton.IconSource>
</ui:CommandBarButton>
</ui:CommandBarFlyout.SecondaryCommands>
</ui:CommandBarFlyout>
</controls:UserControlBase.Resources>

7
StabilityMatrix.Avalonia/Views/InferencePage.axaml.cs

@ -60,4 +60,11 @@ public partial class InferencePage : UserControlBase
{
(DataContext as InferenceViewModel)!.AddTabCommand.Execute(InferenceProjectType.Upscale);
}
private void AddTabMenu_ImgToVideo_OnClick(object? sender, RoutedEventArgs e)
{
(DataContext as InferenceViewModel)!.AddTabCommand.Execute(
InferenceProjectType.ImageToVideo
);
}
}

2
StabilityMatrix.Core/Models/Api/Comfy/NodeTypes/NodeConnections.cs

@ -17,3 +17,5 @@ public class ConditioningNodeConnection : NodeConnectionBase { }
public class ClipNodeConnection : NodeConnectionBase { }
public class ControlNetNodeConnection : NodeConnectionBase { }
public class ClipVisionNodeConnection : NodeConnectionBase { }

41
StabilityMatrix.Core/Models/Api/Comfy/Nodes/ComfyNodeBuilder.cs

@ -290,6 +290,12 @@ public class ComfyNodeBuilder
public required string CkptName { get; init; }
}
public record ImageOnlyCheckpointLoader
: ComfyTypedNodeBase<ModelNodeConnection, ClipVisionNodeConnection, VAENodeConnection>
{
public required string CkptName { get; init; }
}
public record FreeU : ComfyTypedNodeBase<ModelNodeConnection>
{
public required ModelNodeConnection Model { get; init; }
@ -357,6 +363,40 @@ public class ComfyNodeBuilder
public required double EndPercent { get; init; }
}
public record SVD_img2vid_Conditioning
: ComfyTypedNodeBase<
ConditioningNodeConnection,
ConditioningNodeConnection,
LatentNodeConnection
>
{
public required ClipVisionNodeConnection ClipVision { get; init; }
public required ImageNodeConnection InitImage { get; init; }
public required VAENodeConnection Vae { get; init; }
public required int Width { get; init; }
public required int Height { get; init; }
public required int VideoFrames { get; init; }
public required int MotionBucketId { get; init; }
public required int Fps { get; set; }
public required double AugmentationLevel { get; init; }
}
public record VideoLinearCFGGuidance : ComfyTypedNodeBase<ModelNodeConnection>
{
public required ModelNodeConnection Model { get; init; }
public required double MinCfg { get; init; }
}
public record SaveAnimatedWEBP : ComfyTypedNodeBase
{
public required ImageNodeConnection Images { get; init; }
public required string FilenamePrefix { get; init; }
public required double Fps { get; init; }
public required bool Lossless { get; init; }
public required int Quality { get; init; }
public required string Method { get; init; }
}
public ImageNodeConnection Lambda_LatentToImage(
LatentNodeConnection latent,
VAENodeConnection vae
@ -823,6 +863,7 @@ public class ComfyNodeBuilder
public ModelNodeConnection? BaseModel { get; set; }
public VAENodeConnection? BaseVAE { get; set; }
public ClipNodeConnection? BaseClip { get; set; }
public ClipVisionNodeConnection? BaseClipVision { get; set; }
public ConditioningNodeConnection? BaseConditioning { get; set; }
public ConditioningNodeConnection? BaseNegativeConditioning { get; set; }

Loading…
Cancel
Save