From 7fba28fd0ae809875daeed5e94ab706c8c54721e Mon Sep 17 00:00:00 2001 From: Ionite Date: Thu, 16 Nov 2023 22:33:28 -0500 Subject: [PATCH 01/30] Add update channel selection and V3 update format support --- .../ViewModels/Dialogs/UpdateViewModel.cs | 2 +- .../Models/Settings/Settings.cs | 9 +- .../Models/Update/UpdateCollection.cs | 11 -- .../Models/Update/UpdateInfo.cs | 83 +++++---- .../Models/Update/UpdateManifest.cs | 17 ++ .../Models/Update/UpdatePlatforms.cs | 28 +++ StabilityMatrix.Core/Updater/UpdateHelper.cs | 161 ++++++++---------- .../ViewModels/UpdateWindowViewModel.cs | 45 +++-- 8 files changed, 206 insertions(+), 150 deletions(-) delete mode 100644 StabilityMatrix.Core/Models/Update/UpdateCollection.cs create mode 100644 StabilityMatrix.Core/Models/Update/UpdateManifest.cs create mode 100644 StabilityMatrix.Core/Models/Update/UpdatePlatforms.cs diff --git a/StabilityMatrix.Avalonia/ViewModels/Dialogs/UpdateViewModel.cs b/StabilityMatrix.Avalonia/ViewModels/Dialogs/UpdateViewModel.cs index 95fe270b..d33cb273 100644 --- a/StabilityMatrix.Avalonia/ViewModels/Dialogs/UpdateViewModel.cs +++ b/StabilityMatrix.Avalonia/ViewModels/Dialogs/UpdateViewModel.cs @@ -124,7 +124,7 @@ public partial class UpdateViewModel : ContentDialogViewModelBase if (UpdateInfo is null) return; - ReleaseNotes = await GetReleaseNotes(UpdateInfo.ChangelogUrl); + ReleaseNotes = await GetReleaseNotes(UpdateInfo.Changelog.ToString()); } internal async Task GetReleaseNotes(string changelogUrl) diff --git a/StabilityMatrix.Core/Models/Settings/Settings.cs b/StabilityMatrix.Core/Models/Settings/Settings.cs index 8997f144..19c65f98 100644 --- a/StabilityMatrix.Core/Models/Settings/Settings.cs +++ b/StabilityMatrix.Core/Models/Settings/Settings.cs @@ -1,8 +1,8 @@ -using System.Drawing; -using System.Globalization; +using System.Globalization; using System.Text.Json.Serialization; using Semver; using StabilityMatrix.Core.Converters.Json; +using StabilityMatrix.Core.Models.Update; namespace StabilityMatrix.Core.Models.Settings; @@ -38,6 +38,11 @@ public class Settings public string? WebApiHost { get; set; } public string? WebApiPort { get; set; } + /// + /// Preferred update channel + /// + public UpdateChannel PreferredUpdateChannel { get; set; } = UpdateChannel.Stable; + /// /// The last auto-update version that had a notification dismissed by the user /// diff --git a/StabilityMatrix.Core/Models/Update/UpdateCollection.cs b/StabilityMatrix.Core/Models/Update/UpdateCollection.cs deleted file mode 100644 index 46b6cc93..00000000 --- a/StabilityMatrix.Core/Models/Update/UpdateCollection.cs +++ /dev/null @@ -1,11 +0,0 @@ -using System.Text.Json.Serialization; - -namespace StabilityMatrix.Core.Models.Update; - -public record UpdateCollection ( - [property: JsonPropertyName("win-x64")] - UpdateInfo? WindowsX64, - - [property: JsonPropertyName("linux-x64")] - UpdateInfo? LinuxX64 -); diff --git a/StabilityMatrix.Core/Models/Update/UpdateInfo.cs b/StabilityMatrix.Core/Models/Update/UpdateInfo.cs index b0470c85..4e11d891 100644 --- a/StabilityMatrix.Core/Models/Update/UpdateInfo.cs +++ b/StabilityMatrix.Core/Models/Update/UpdateInfo.cs @@ -1,37 +1,58 @@ -using System.Diagnostics.CodeAnalysis; +using System.Globalization; using System.Text.Json.Serialization; using Semver; using StabilityMatrix.Core.Converters.Json; +using StabilityMatrix.Core.Extensions; namespace StabilityMatrix.Core.Models.Update; -[SuppressMessage("ReSharper", "ClassNeverInstantiated.Global")] -public record UpdateInfo( - [property: JsonPropertyName("version"), JsonConverter(typeof(SemVersionJsonConverter))] - SemVersion Version, - - [property: JsonPropertyName("releaseDate")] - DateTimeOffset ReleaseDate, - - [property: JsonPropertyName("channel")] - UpdateChannel Channel, - - [property: JsonPropertyName("type")] - UpdateType Type, - - [property: JsonPropertyName("url")] - string DownloadUrl, - - [property: JsonPropertyName("changelog")] - string ChangelogUrl, - - // Blake3 hash of the file - [property: JsonPropertyName("hashBlake3")] - string HashBlake3, - - // ED25519 signature of the semicolon seperated string: - // "version + releaseDate + channel + type + url + changelog + hash_blake3" - // verifiable using our stored public key - [property: JsonPropertyName("signature")] - string Signature -); +public record UpdateInfo +{ + [JsonConverter(typeof(SemVersionJsonConverter))] + public required SemVersion Version { get; init; } + + public required DateTimeOffset ReleaseDate { get; init; } + + public UpdateChannel Channel { get; init; } + + public UpdateType Type { get; init; } + + public required Uri Url { get; init; } + + public required Uri Changelog { get; init; } + + /// + /// Blake3 hash of the file + /// + public required string HashBlake3 { get; init; } + + /// + /// ED25519 signature of the semicolon seperated string: + /// "version + releaseDate + channel + type + url + changelog + hash_blake3" + /// verifiable using our stored public key + /// + public required string Signature { get; init; } + + /// + /// Data for use in signature verification. + /// Semicolon separated string of fields: + /// "version, releaseDate, channel, type, url, changelog, hashBlake3" + /// + public string GetSignedData() + { + var channel = Channel.GetStringValue().ToLowerInvariant(); + var date = FormatDateTimeOffsetInvariant(ReleaseDate); + return $"{Version};{date};{channel};" + $"{(int)Type};{Url};{Changelog};" + $"{HashBlake3}"; + } + + /// + /// Format a DatetimeOffset to a culture invariant string for use in signature verification. + /// + private static string FormatDateTimeOffsetInvariant(DateTimeOffset dateTimeOffset) + { + return dateTimeOffset.ToString( + @"yyyy-MM-ddTHH\:mm\:ss.ffffffzzz", + CultureInfo.InvariantCulture + ); + } +} diff --git a/StabilityMatrix.Core/Models/Update/UpdateManifest.cs b/StabilityMatrix.Core/Models/Update/UpdateManifest.cs new file mode 100644 index 00000000..e4be6f15 --- /dev/null +++ b/StabilityMatrix.Core/Models/Update/UpdateManifest.cs @@ -0,0 +1,17 @@ +using System.Text.Json.Serialization; + +namespace StabilityMatrix.Core.Models.Update; + +[JsonSerializable(typeof(UpdateManifest))] +public record UpdateManifest +{ + public required Dictionary Updates { get; init; } +} + + +// TODO: Bugged in .NET 7 but we can use in 8 https://github.com/dotnet/runtime/pull/79828 +/*[JsonSourceGenerationOptions(PropertyNamingPolicy = JsonKnownNamingPolicy.CamelCase)] +[JsonSerializable(typeof(UpdateManifest))] +public partial class UpdateManifestContext : JsonSerializerContext +{ +}*/ diff --git a/StabilityMatrix.Core/Models/Update/UpdatePlatforms.cs b/StabilityMatrix.Core/Models/Update/UpdatePlatforms.cs new file mode 100644 index 00000000..b6711cd5 --- /dev/null +++ b/StabilityMatrix.Core/Models/Update/UpdatePlatforms.cs @@ -0,0 +1,28 @@ +using System.Text.Json.Serialization; +using StabilityMatrix.Core.Helper; + +namespace StabilityMatrix.Core.Models.Update; + +public record UpdatePlatforms +{ + [JsonPropertyName("win-x64")] + public UpdateInfo? WindowsX64 { get; init; } + + [JsonPropertyName("linux-x64")] + public UpdateInfo? LinuxX64 { get; init; } + + public UpdateInfo? GetInfoForCurrentPlatform() + { + if (Compat.IsWindows) + { + return WindowsX64; + } + + if (Compat.IsLinux) + { + return LinuxX64; + } + + return null; + } +} diff --git a/StabilityMatrix.Core/Updater/UpdateHelper.cs b/StabilityMatrix.Core/Updater/UpdateHelper.cs index 325af086..f990a81c 100644 --- a/StabilityMatrix.Core/Updater/UpdateHelper.cs +++ b/StabilityMatrix.Core/Updater/UpdateHelper.cs @@ -1,9 +1,7 @@ -using System.Globalization; -using System.Text.Json; +using System.Text.Json; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using StabilityMatrix.Core.Attributes; -using StabilityMatrix.Core.Extensions; using StabilityMatrix.Core.Helper; using StabilityMatrix.Core.Models.Configs; using StabilityMatrix.Core.Models.FileInterfaces; @@ -19,11 +17,12 @@ public class UpdateHelper : IUpdateHelper private readonly ILogger logger; private readonly IHttpClientFactory httpClientFactory; private readonly IDownloadService downloadService; + private readonly ISettingsManager settingsManager; private readonly DebugOptions debugOptions; private readonly System.Timers.Timer timer = new(TimeSpan.FromMinutes(60)); private string UpdateManifestUrl => - debugOptions.UpdateManifestUrl ?? "https://cdn.lykos.ai/update-v2.json"; + debugOptions.UpdateManifestUrl ?? "https://cdn.lykos.ai/update-v3.json"; public const string UpdateFolderName = ".StabilityMatrixUpdate"; public static DirectoryPath UpdateFolder => Compat.AppCurrentDir.JoinDir(UpdateFolderName); @@ -34,12 +33,14 @@ public class UpdateHelper : IUpdateHelper ILogger logger, IHttpClientFactory httpClientFactory, IDownloadService downloadService, - IOptions debugOptions + IOptions debugOptions, + ISettingsManager settingsManager ) { this.logger = logger; this.httpClientFactory = httpClientFactory; this.downloadService = downloadService; + this.settingsManager = settingsManager; this.debugOptions = debugOptions.Value; timer.Elapsed += async (_, _) => @@ -57,15 +58,13 @@ public class UpdateHelper : IUpdateHelper public async Task DownloadUpdate(UpdateInfo updateInfo, IProgress progress) { - var downloadUrl = updateInfo.DownloadUrl; - UpdateFolder.Create(); UpdateFolder.Info.Attributes |= FileAttributes.Hidden; // download the file from URL await downloadService .DownloadToFileAsync( - downloadUrl, + updateInfo.Url.ToString(), ExecutablePath, progress: progress, httpClientName: "UpdateClient" @@ -73,31 +72,6 @@ public class UpdateHelper : IUpdateHelper .ConfigureAwait(false); } - /// - /// Format a DatetimeOffset to a culture invariant string for use in signature verification. - /// - private static string FormatDateTimeOffsetInvariant(DateTimeOffset dateTimeOffset) - { - return dateTimeOffset.ToString( - @"yyyy-MM-ddTHH\:mm\:ss.ffffffzzz", - CultureInfo.InvariantCulture - ); - } - - /// - /// Data for use in signature verification. - /// Semicolon separated string of fields: - /// "version, releaseDate, channel, type, url, changelog, hashBlake3" - /// - private static string GetUpdateInfoSignedData(UpdateInfo updateInfo) - { - var channel = updateInfo.Channel.GetStringValue().ToLowerInvariant(); - var date = FormatDateTimeOffsetInvariant(updateInfo.ReleaseDate); - return $"{updateInfo.Version};{date};{channel};" - + $"{(int)updateInfo.Type};{updateInfo.DownloadUrl};{updateInfo.ChangelogUrl};" - + $"{updateInfo.HashBlake3}"; - } - private async Task CheckForUpdate() { try @@ -114,85 +88,94 @@ public class UpdateHelper : IUpdateHelper return; } - var updateCollection = await JsonSerializer - .DeserializeAsync( - await response.Content.ReadAsStreamAsync().ConfigureAwait(false) + var updateManifest = await JsonSerializer + .DeserializeAsync( + await response.Content.ReadAsStreamAsync().ConfigureAwait(false), + new JsonSerializerOptions { PropertyNamingPolicy = JsonNamingPolicy.CamelCase } ) .ConfigureAwait(false); - if (updateCollection is null) + if (updateManifest is null) { - logger.LogError("UpdateCollection is null"); + logger.LogError("UpdateManifest is null"); return; } - // Get the update info for our platform - var updateInfo = updateCollection switch - { - _ when Compat.IsWindows && Compat.IsX64 => updateCollection.WindowsX64, - _ when Compat.IsLinux && Compat.IsX64 => updateCollection.LinuxX64, - _ => null - }; - - if (updateInfo is null) + foreach ( + var channel in Enum.GetValues(typeof(UpdateChannel)) + .Cast() + .Where(c => c > UpdateChannel.Unknown) + ) { - logger.LogWarning( - "Could not find compatible update info for the platform {Platform}", - Compat.Platform - ); - return; + if ( + updateManifest.Updates.TryGetValue(channel, out var platforms) + && platforms.GetInfoForCurrentPlatform() is { } update + && ValidateUpdate(update) + ) + { + NotifyUpdateAvailable(update); + return; + } } - logger.LogInformation("UpdateInfo signature: {Signature}", updateInfo.Signature); + logger.LogInformation("No update available"); + } + catch (Exception e) + { + logger.LogError(e, "Couldn't check for update"); + } + } - var updateInfoSignData = GetUpdateInfoSignedData(updateInfo); - logger.LogInformation("UpdateInfo signed data: {SignData}", updateInfoSignData); + private bool ValidateUpdate(UpdateInfo? update) + { + if (update is null) + return false; - // Verify signature - var checker = new SignatureChecker(); - if (!checker.Verify(updateInfoSignData, updateInfo.Signature)) - { - logger.LogError("UpdateInfo signature is invalid: {Info}", updateInfo); - return; - } - logger.LogInformation("UpdateInfo signature verified"); + // Verify signature + var checker = new SignatureChecker(); + var signedData = update.GetSignedData(); - var order = updateInfo.Version.ComparePrecedenceTo(Compat.AppVersion); + if (!checker.Verify(signedData, update.Signature)) + { + logger.LogError( + "UpdateInfo signature {Signature} is invalid, Data = {Data}, UpdateInfo = {Info}", + update.Signature, + signedData, + update + ); + return false; + } - if (order > 0) - { + switch (update.Version.ComparePrecedenceTo(Compat.AppVersion)) + { + case > 0: // Newer version available - logger.LogInformation( - "Update available {AppVer} -> {UpdateVer}", - Compat.AppVersion, - updateInfo.Version - ); - EventManager.Instance.OnUpdateAvailable(updateInfo); - return; - } - if (order == 0) + return true; + case 0: { // Same version available, check if we both have commit hash metadata - var updateHash = updateInfo.Version.Metadata; + var updateHash = update.Version.Metadata; var appHash = Compat.AppVersion.Metadata; // If different, we can update if (updateHash != appHash) { - logger.LogInformation( - "Update available {AppVer} -> {UpdateVer}", - Compat.AppVersion, - updateInfo.Version - ); - EventManager.Instance.OnUpdateAvailable(updateInfo); - return; + return true; } - } - logger.LogInformation("No update available"); - } - catch (Exception e) - { - logger.LogError(e, "Couldn't check for update"); + break; + } } + + return false; + } + + private void NotifyUpdateAvailable(UpdateInfo update) + { + logger.LogInformation( + "Update available {AppVer} -> {UpdateVer}", + Compat.AppVersion, + update.Version + ); + EventManager.Instance.OnUpdateAvailable(update); } } diff --git a/StabilityMatrix/ViewModels/UpdateWindowViewModel.cs b/StabilityMatrix/ViewModels/UpdateWindowViewModel.cs index c5d95229..d61a8295 100644 --- a/StabilityMatrix/ViewModels/UpdateWindowViewModel.cs +++ b/StabilityMatrix/ViewModels/UpdateWindowViewModel.cs @@ -19,28 +19,38 @@ public partial class UpdateWindowViewModel : ObservableObject private readonly IHttpClientFactory httpClientFactory; private readonly IUpdateHelper updateHelper; - public UpdateWindowViewModel(ISettingsManager settingsManager, - IHttpClientFactory httpClientFactory, IUpdateHelper updateHelper) + public UpdateWindowViewModel( + ISettingsManager settingsManager, + IHttpClientFactory httpClientFactory, + IUpdateHelper updateHelper + ) { this.settingsManager = settingsManager; this.httpClientFactory = httpClientFactory; this.updateHelper = updateHelper; } - [ObservableProperty] private string? releaseNotes; - [ObservableProperty] private string? updateText; - [ObservableProperty] private int progressValue; - [ObservableProperty] private bool showProgressBar; - + [ObservableProperty] + private string? releaseNotes; + + [ObservableProperty] + private string? updateText; + + [ObservableProperty] + private int progressValue; + + [ObservableProperty] + private bool showProgressBar; public UpdateInfo? UpdateInfo { get; set; } public async Task OnLoaded() { - UpdateText = $"Stability Matrix v{UpdateInfo?.Version} is now available! You currently have v{Utilities.GetAppVersion()}. Would you like to update now?"; - + UpdateText = + $"Stability Matrix v{UpdateInfo?.Version} is now available! You currently have v{Utilities.GetAppVersion()}. Would you like to update now?"; + var client = httpClientFactory.CreateClient(); - var response = await client.GetAsync(UpdateInfo?.ChangelogUrl); + var response = await client.GetAsync(UpdateInfo?.Changelog); if (response.IsSuccessStatusCode) { ReleaseNotes = await response.Content.ReadAsStringAsync(); @@ -58,14 +68,17 @@ public partial class UpdateWindowViewModel : ObservableObject { return; } - + ShowProgressBar = true; UpdateText = $"Downloading update v{UpdateInfo.Version}..."; - await updateHelper.DownloadUpdate(UpdateInfo, new Progress(report => - { - ProgressValue = Convert.ToInt32(report.Percentage); - })); - + await updateHelper.DownloadUpdate( + UpdateInfo, + new Progress(report => + { + ProgressValue = Convert.ToInt32(report.Percentage); + }) + ); + UpdateText = "Update complete. Restarting Stability Matrix in 3 seconds..."; await Task.Delay(1000); UpdateText = "Update complete. Restarting Stability Matrix in 2 seconds..."; From 0e9112617842c77f31bbc16fbc690dce30cfa0bf Mon Sep 17 00:00:00 2001 From: Ionite Date: Thu, 16 Nov 2023 22:37:51 -0500 Subject: [PATCH 02/30] Fix UI tests for updates --- StabilityMatrix.UITests/TestAppBuilder.cs | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/StabilityMatrix.UITests/TestAppBuilder.cs b/StabilityMatrix.UITests/TestAppBuilder.cs index a7bbf3fd..9aca5557 100644 --- a/StabilityMatrix.UITests/TestAppBuilder.cs +++ b/StabilityMatrix.UITests/TestAppBuilder.cs @@ -45,16 +45,18 @@ public static class TestAppBuilder serviceCollection.AddSingleton(settingsManager); // IUpdateHelper - var mockUpdateInfo = new UpdateInfo( - SemVersion.Parse("2.999.0"), - DateTimeOffset.UnixEpoch, - UpdateChannel.Stable, - UpdateType.Normal, - "https://example.org", - "https://example.org", - "46e11a5216c55d4c9d3c54385f62f3e1022537ae191615237f05e06d6f8690d0", - "IX5/CCXWJQG0oGkYWVnuF34gTqF/dJSrDrUd6fuNMYnncL39G3HSvkXrjvJvR18MA2rQNB5z13h3/qBSf9c7DA==" - ); + var mockUpdateInfo = new UpdateInfo() + { + Version = SemVersion.Parse("2.999.0"), + ReleaseDate = DateTimeOffset.UnixEpoch, + Channel = UpdateChannel.Stable, + Type = UpdateType.Normal, + Url = new Uri("https://example.org"), + Changelog = new Uri("https://example.org"), + HashBlake3 = "46e11a5216c55d4c9d3c54385f62f3e1022537ae191615237f05e06d6f8690d0", + Signature = + "IX5/CCXWJQG0oGkYWVnuF34gTqF/dJSrDrUd6fuNMYnncL39G3HSvkXrjvJvR18MA2rQNB5z13h3/qBSf9c7DA==" + }; var updateHelper = Substitute.For(); updateHelper From 0bb28ed43e8a3a2914ae143b3c28a41b70f9c13e Mon Sep 17 00:00:00 2001 From: Ionite Date: Fri, 17 Nov 2023 13:12:07 -0500 Subject: [PATCH 03/30] Add auto update release CI --- .github/workflows/release.yml | 96 +++++++++++++++++++++++++++++++++++ 1 file changed, 96 insertions(+) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index b7d84081..a6540d5b 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -26,6 +26,23 @@ on: type: boolean description: Mark GitHub Release as Prerelease? default: false + auto-update-release: + type: boolean + description: Release auto-update? + default: false + auto-update-release-mode: + type: choice + description: Release auto-update mode + options: + - github url + - upload to b2 + auto-update-release-channel: + type: choice + description: Release auto-update channel + options: + - stable + - preview + - development jobs: release-linux: @@ -199,3 +216,82 @@ jobs: body: ${{ steps.release_notes.outputs.release_notes }} draft: ${{ github.event.inputs.github-release-draft == 'true' }} prerelease: ${{ github.event.inputs.github-release-prerelease == 'true' }} + + publish-auto-update-github: + name: Publish Auto-Update Release (GitHub) + needs: [ release-linux, release-windows ] + if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.auto-update-release == 'true' && github.event.inputs.auto-update-release-mode == 'github url' }} + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Set Version from manual input + run: | + echo "Using version ${{ github.event.inputs.version }}" + echo "RELEASE_VERSION=${{ github.event.inputs.version }}" >> $env:GITHUB_ENV + + - uses: actions/setup-python@v4 + with: + python-version: '3.11' + + - name: Install Python Dependencies + run: pip install stability-matrix-tools + + - name: Publish Auto-Update Release + run: sm-tools updates publish-matrix -v $RELEASE_VERSION -y + + publish-auto-update-b2: + name: Publish Auto-Update Release (B2) + needs: [ release-linux, release-windows ] + if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.auto-update-release == 'true' && github.event.inputs.auto-update-release-mode == 'upload to b2' }} + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Set Version from manual input + run: | + echo "Using version ${{ github.event.inputs.version }}" + echo "RELEASE_VERSION=${{ github.event.inputs.version }}" >> $env:GITHUB_ENV + + - uses: actions/setup-python@v4 + with: + python-version: '3.11' + + - name: Install Python Dependencies + run: pip install stability-matrix-tools + + - name: Download Changelog + run: > + sm-tools updates download-changelog -v $RELEASE_VERSION -y + --changelog + + # Zip each build + - name: Zip Artifacts + run: | + zip -r StabilityMatrix-win-x64.zip StabilityMatrix-win-x64/* + zip -r StabilityMatrix-linux-x64.zip StabilityMatrix-linux-x64/* + + # Check that the zips and CHANGELOG.md are in the current working directory + - name: Check files + run: | + if [ ! -f StabilityMatrix-win-x64.zip ]; then + echo "StabilityMatrix-win-x64.zip not found" + exit 1 + fi + if [ ! -f StabilityMatrix-linux-x64.zip ]; then + echo "StabilityMatrix-linux-x64.zip not found" + exit 1 + fi + if [ ! -f CHANGELOG.md ]; then + echo "CHANGELOG.md not found" + exit 1 + fi + + - name: Publish Auto-Update Release + run: > + sm-tools updates publish-files-v3 -v $RELEASE_VERSION -y + --channel ${{ github.event.inputs.auto-update-release-channel }} + --changelog CHANGELOG.md + --win-x64 StabilityMatrix-win-x64.zip + --linux-x64 StabilityMatrix-linux-x64.zip + --b2-bucket-name ${{ secrets.B2_BUCKET_NAME }} From 48701d836087178dfeea24ecae9ec9a70f8afe57 Mon Sep 17 00:00:00 2001 From: Ionite Date: Fri, 17 Nov 2023 13:13:24 -0500 Subject: [PATCH 04/30] Specify sm-tools versions --- .github/workflows/release.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index a6540d5b..0aa23972 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -235,7 +235,7 @@ jobs: python-version: '3.11' - name: Install Python Dependencies - run: pip install stability-matrix-tools + run: pip install stability-matrix-tools~=0.2.7 - name: Publish Auto-Update Release run: sm-tools updates publish-matrix -v $RELEASE_VERSION -y @@ -258,7 +258,7 @@ jobs: python-version: '3.11' - name: Install Python Dependencies - run: pip install stability-matrix-tools + run: pip install stability-matrix-tools~=0.2.7 - name: Download Changelog run: > From 3d90e64235f29e662239eea5b3262e3d76a9d434 Mon Sep 17 00:00:00 2001 From: Ionite Date: Fri, 17 Nov 2023 14:17:03 -0500 Subject: [PATCH 05/30] Add updater markdown parse support for channel preferences --- .../ViewModels/Dialogs/UpdateViewModel.cs | 56 ++++++++++++++--- .../Avalonia/UpdateViewModelTests.cs | 60 +++++++++++++++++++ 2 files changed, 107 insertions(+), 9 deletions(-) diff --git a/StabilityMatrix.Avalonia/ViewModels/Dialogs/UpdateViewModel.cs b/StabilityMatrix.Avalonia/ViewModels/Dialogs/UpdateViewModel.cs index d33cb273..d83426d4 100644 --- a/StabilityMatrix.Avalonia/ViewModels/Dialogs/UpdateViewModel.cs +++ b/StabilityMatrix.Avalonia/ViewModels/Dialogs/UpdateViewModel.cs @@ -57,7 +57,7 @@ public partial class UpdateViewModel : ContentDialogViewModelBase private string? newVersionText; [GeneratedRegex( - @"(##\s*(v[0-9]+\.[0-9]+\.[0-9]+)((?:\n|.)+?))(?=(##\s*v[0-9]+\.[0-9]+\.[0-9]+)|\z)" + @"(##\s*(v[0-9]+\.[0-9]+\.[0-9]+(?:-(?:[0-9A-Za-z-.]+))?)((?:\n|.)+?))(?=(##\s*v[0-9]+\.[0-9]+\.[0-9]+)|\z)" )] private static partial Regex RegexChangelog(); @@ -82,7 +82,14 @@ public partial class UpdateViewModel : ContentDialogViewModelBase /// /// Formats changelog markdown including up to the current version /// - internal static string? FormatChangelog(string markdown, SemVersion currentVersion) + /// Markdown to format + /// Versions equal or below this are excluded + /// Maximum channel level to include + internal static string? FormatChangelog( + string markdown, + SemVersion currentVersion, + UpdateChannel maxChannel = UpdateChannel.Stable + ) { var pattern = RegexChangelog(); @@ -93,28 +100,59 @@ public partial class UpdateViewModel : ContentDialogViewModelBase new { Block = m.Groups[1].Value.Trim(), - Version = m.Groups[2].Value.Trim(), + Version = SemVersion.TryParse( + m.Groups[2].Value.Trim(), + SemVersionStyles.AllowV, + out var version + ) + ? version + : null, Content = m.Groups[3].Value.Trim() } ) + .Where(x => x.Version is not null) .ToList(); // Join all blocks until and excluding the current version // If we're on a pre-release, include the current release - var currentVersionBlock = results.FindIndex( - x => x.Version == $"v{currentVersion.WithoutPrereleaseOrMetadata()}" + x => x.Version == currentVersion.WithoutMetadata() ); + // Support for previous pre-release without changelogs + if (currentVersionBlock == -1) + { + currentVersionBlock = results.FindIndex( + x => x.Version == currentVersion.WithoutPrereleaseOrMetadata() + ); + + // Add 1 if found to include the current release + if (currentVersionBlock != -1) + { + currentVersionBlock++; + } + } + + // Still not found, just include all if (currentVersionBlock == -1) { - return null; + currentVersionBlock = results.Count; } + // Filter out pre-releases var blocks = results - .Take(currentVersionBlock + (currentVersion.IsPrerelease ? 1 : 0)) - .Select(x => x.Block) - .ToList(); + .Take(currentVersionBlock) + .Where( + x => + x.Version!.PrereleaseIdentifiers.Count == 0 + || x.Version.PrereleaseIdentifiers[0].Value switch + { + "pre" when maxChannel >= UpdateChannel.Preview => true, + "dev" when maxChannel >= UpdateChannel.Development => true, + _ => false + } + ) + .Select(x => x.Block); return string.Join(Environment.NewLine + Environment.NewLine, blocks); } diff --git a/StabilityMatrix.Tests/Avalonia/UpdateViewModelTests.cs b/StabilityMatrix.Tests/Avalonia/UpdateViewModelTests.cs index 7c5bb4e0..829e0b99 100644 --- a/StabilityMatrix.Tests/Avalonia/UpdateViewModelTests.cs +++ b/StabilityMatrix.Tests/Avalonia/UpdateViewModelTests.cs @@ -1,5 +1,6 @@ using Semver; using StabilityMatrix.Avalonia.ViewModels.Dialogs; +using StabilityMatrix.Core.Models.Update; namespace StabilityMatrix.Tests.Avalonia; @@ -65,4 +66,63 @@ public class UpdateViewModelTests """; Assert.AreEqual(expectedPre, resultPre); } + + [TestMethod] + public void FormatChangelogWithChannelTest() + { + // Arrange + const string markdown = """ + # Changelog + + All notable changes to Stability Matrix will be documented in this file. + + The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), + and this project adheres to [Semantic Versioning 2.0](https://semver.org/spec/v2.0.0.html). + + ## v2.4.6 + ### Added + - Stuff + ### Changed + - Things + + ## v2.4.6-pre.1 + ### Fixed + - Fixed bug + + ## v2.4.6-dev.1 + ### Fixed + - Fixed bug + + ## v2.4.5 + ### Changed + - Changed stuff + """; + + // Act + var result = UpdateViewModel.FormatChangelog( + markdown, + SemVersion.Parse("2.4.0"), + UpdateChannel.Preview + ); + + // Assert + const string expected = """ + ## v2.4.6 + ### Added + - Stuff + ### Changed + - Things + + ## v2.4.6-pre.1 + ### Fixed + - Fixed bug + + ## v2.4.5 + ### Changed + - Changed stuff + """; + + // Should include pre but not dev + Assert.AreEqual(expected, result); + } } From 457872f01e78a74b5849f22c4b16c24455798bcf Mon Sep 17 00:00:00 2001 From: Ionite Date: Fri, 17 Nov 2023 16:38:50 -0500 Subject: [PATCH 06/30] Change MoveToWithIncrement as separate method --- .../ViewModels/OutputsPageViewModel.cs | 2 +- .../Models/FileInterfaces/FilePath.cs | 69 ++++++++++++++----- 2 files changed, 54 insertions(+), 17 deletions(-) diff --git a/StabilityMatrix.Avalonia/ViewModels/OutputsPageViewModel.cs b/StabilityMatrix.Avalonia/ViewModels/OutputsPageViewModel.cs index 8352f6d5..2b845f9e 100644 --- a/StabilityMatrix.Avalonia/ViewModels/OutputsPageViewModel.cs +++ b/StabilityMatrix.Avalonia/ViewModels/OutputsPageViewModel.cs @@ -484,7 +484,7 @@ public partial class OutputsPageViewModel : PageViewModelBase continue; } - await file.MoveToAsync(newPath); + await file.MoveToWithIncrementAsync(newPath); } catch (Exception e) { diff --git a/StabilityMatrix.Core/Models/FileInterfaces/FilePath.cs b/StabilityMatrix.Core/Models/FileInterfaces/FilePath.cs index ffe856df..cd0f6887 100644 --- a/StabilityMatrix.Core/Models/FileInterfaces/FilePath.cs +++ b/StabilityMatrix.Core/Models/FileInterfaces/FilePath.cs @@ -7,7 +7,7 @@ namespace StabilityMatrix.Core.Models.FileInterfaces; [SuppressMessage("ReSharper", "MemberCanBePrivate.Global")] [JsonConverter(typeof(StringJsonConverter))] -public class FilePath : FileSystemPath, IPathObject +public partial class FilePath : FileSystemPath, IPathObject { private FileInfo? _info; @@ -33,6 +33,7 @@ public class FilePath : FileSystemPath, IPathObject [JsonIgnore] public string NameWithoutExtension => Path.GetFileNameWithoutExtension(Info.Name); + /// [JsonIgnore] public string Extension => Info.Extension; @@ -138,6 +139,26 @@ public class FilePath : FileSystemPath, IPathObject return File.WriteAllBytesAsync(FullPath, bytes, ct); } + /// + /// Rename the file. + /// + public FilePath Rename(string fileName) + { + if ( + Path.GetDirectoryName(FullPath) is { } directory + && !string.IsNullOrWhiteSpace(directory) + ) + { + var target = Path.Combine(directory, fileName); + Info.MoveTo(target, true); + return new FilePath(target); + } + + throw new InvalidOperationException( + "Cannot rename a file path that is empty or has no directory" + ); + } + /// /// Move the file to a directory. /// @@ -163,25 +184,41 @@ public class FilePath : FileSystemPath, IPathObject /// public async Task MoveToAsync(FilePath destinationFile) { - await Task.Run(() => - { - var path = destinationFile.FullPath; - if (destinationFile.Exists) - { - var num = Random.Shared.NextInt64(0, 10000); - path = path.Replace( - destinationFile.NameWithoutExtension, - $"{destinationFile.NameWithoutExtension}_{num}" - ); - } - - Info.MoveTo(path); - }) - .ConfigureAwait(false); + await Task.Run(() => Info.MoveTo(destinationFile.FullPath)).ConfigureAwait(false); // Return the new path return destinationFile; } + /// + /// Move the file to a target path with auto increment if the file already exists. + /// + /// The new path, possibly with incremented file name + public async Task MoveToWithIncrementAsync( + FilePath destinationFile, + int maxTries = 100 + ) + { + await Task.Yield(); + + var targetFile = destinationFile; + + for (var i = 1; i < maxTries; i++) + { + if (!targetFile.Exists) + { + return await MoveToAsync(targetFile).ConfigureAwait(false); + } + + targetFile = destinationFile.WithName( + destinationFile.NameWithoutExtension + $" ({i})" + destinationFile.Extension + ); + } + + throw new IOException( + $"Could not move file to {destinationFile} because it already exists." + ); + } + /// /// Copy the file to a target path. /// From fae4add130f5d987b80e648dcab9026a278dd11e Mon Sep 17 00:00:00 2001 From: Ionite Date: Fri, 17 Nov 2023 16:39:21 -0500 Subject: [PATCH 07/30] Add update handling for zip files --- StabilityMatrix.Core/Updater/UpdateHelper.cs | 42 +++++++++++++++----- 1 file changed, 33 insertions(+), 9 deletions(-) diff --git a/StabilityMatrix.Core/Updater/UpdateHelper.cs b/StabilityMatrix.Core/Updater/UpdateHelper.cs index f990a81c..a9751348 100644 --- a/StabilityMatrix.Core/Updater/UpdateHelper.cs +++ b/StabilityMatrix.Core/Updater/UpdateHelper.cs @@ -61,15 +61,39 @@ public class UpdateHelper : IUpdateHelper UpdateFolder.Create(); UpdateFolder.Info.Attributes |= FileAttributes.Hidden; - // download the file from URL - await downloadService - .DownloadToFileAsync( - updateInfo.Url.ToString(), - ExecutablePath, - progress: progress, - httpClientName: "UpdateClient" - ) - .ConfigureAwait(false); + var downloadFile = UpdateFolder.JoinFile(Path.GetFileName(updateInfo.Url.ToString())); + + try + { + // download the file from URL + await downloadService + .DownloadToFileAsync( + updateInfo.Url.ToString(), + downloadFile, + progress: progress, + httpClientName: "UpdateClient" + ) + .ConfigureAwait(false); + + // Unzip if needed + if (downloadFile.Extension == ".zip") + { + await ArchiveHelper + .Extract(downloadFile, UpdateFolder, progress) + .ConfigureAwait(false); + await downloadFile.DeleteAsync().ConfigureAwait(false); + } + // Otherwise just rename + else + { + downloadFile.Rename(ExecutablePath.Name); + } + } + finally + { + // Clean up original download + await downloadFile.DeleteAsync().ConfigureAwait(false); + } } private async Task CheckForUpdate() From 001c5a84ea2368be3c8edf3d6730e67524ab0047 Mon Sep 17 00:00:00 2001 From: Ionite Date: Fri, 17 Nov 2023 16:42:50 -0500 Subject: [PATCH 08/30] Update chagenlog --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ed086a33..28a6b5be 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,10 @@ All notable changes to Stability Matrix will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning 2.0](https://semver.org/spec/v2.0.0.html). +## v2.6.2 +### Changed +- Backend changes for auto-update schema v3, supporting customizable release channels and faster downloads with zip compression + ## v2.6.1 ### Changed - NVIDIA GPU users will be updated to use CUDA 12.1 for the InvokeAI package for a slight performance improvement From 20f2f9784adb15f6773d4fce1100574bb9cafc8e Mon Sep 17 00:00:00 2001 From: Ionite Date: Fri, 17 Nov 2023 16:46:04 -0500 Subject: [PATCH 09/30] oops forgot to push a file --- .../Models/FileInterfaces/FilePath.Fluent.cs | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 StabilityMatrix.Core/Models/FileInterfaces/FilePath.Fluent.cs diff --git a/StabilityMatrix.Core/Models/FileInterfaces/FilePath.Fluent.cs b/StabilityMatrix.Core/Models/FileInterfaces/FilePath.Fluent.cs new file mode 100644 index 00000000..9a27f46a --- /dev/null +++ b/StabilityMatrix.Core/Models/FileInterfaces/FilePath.Fluent.cs @@ -0,0 +1,20 @@ +namespace StabilityMatrix.Core.Models.FileInterfaces; + +public partial class FilePath +{ + /// + /// Return a new with the given file name. + /// + public FilePath WithName(string fileName) + { + if ( + Path.GetDirectoryName(FullPath) is { } directory + && !string.IsNullOrWhiteSpace(directory) + ) + { + return new FilePath(directory, fileName); + } + + return new FilePath(fileName); + } +} From 4002a24d404032febb54323139e9021f56c944f4 Mon Sep 17 00:00:00 2001 From: Ionite Date: Fri, 17 Nov 2023 16:59:40 -0500 Subject: [PATCH 10/30] Fix zip nested folders in github release CI --- .github/workflows/release.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 0aa23972..44a9426f 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -197,8 +197,8 @@ jobs: # Zip each build - name: Zip Artifacts run: | - zip -r StabilityMatrix-win-x64.zip StabilityMatrix-win-x64 - zip -r StabilityMatrix-linux-x64.zip StabilityMatrix-linux-x64 + zip -r StabilityMatrix-win-x64.zip StabilityMatrix-win-x64/* + zip -r StabilityMatrix-linux-x64.zip StabilityMatrix-linux-x64/* - name: Create Github Release id: create_release From ed0f53bd1d19a6ac18682f75fc6b1cc977601fc0 Mon Sep 17 00:00:00 2001 From: Ionite Date: Fri, 17 Nov 2023 17:07:02 -0500 Subject: [PATCH 11/30] Add nested zip path handling --- StabilityMatrix.Core/Updater/UpdateHelper.cs | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/StabilityMatrix.Core/Updater/UpdateHelper.cs b/StabilityMatrix.Core/Updater/UpdateHelper.cs index a9751348..d67e9bfe 100644 --- a/StabilityMatrix.Core/Updater/UpdateHelper.cs +++ b/StabilityMatrix.Core/Updater/UpdateHelper.cs @@ -62,6 +62,7 @@ public class UpdateHelper : IUpdateHelper UpdateFolder.Info.Attributes |= FileAttributes.Hidden; var downloadFile = UpdateFolder.JoinFile(Path.GetFileName(updateInfo.Url.ToString())); + var extractDir = UpdateFolder.JoinDir("extract"); try { @@ -78,10 +79,18 @@ public class UpdateHelper : IUpdateHelper // Unzip if needed if (downloadFile.Extension == ".zip") { + await extractDir.DeleteAsync().ConfigureAwait(false); + extractDir.Create(); + await ArchiveHelper - .Extract(downloadFile, UpdateFolder, progress) + .Extract(downloadFile, extractDir, progress) .ConfigureAwait(false); - await downloadFile.DeleteAsync().ConfigureAwait(false); + + // Move all (possibly nested) loose files to the root UpdateFolder + foreach (var file in extractDir.EnumerateFiles("*.*", SearchOption.AllDirectories)) + { + await file.MoveToDirectoryAsync(UpdateFolder).ConfigureAwait(false); + } } // Otherwise just rename else @@ -93,6 +102,8 @@ public class UpdateHelper : IUpdateHelper { // Clean up original download await downloadFile.DeleteAsync().ConfigureAwait(false); + // Clean up extract dir + await extractDir.DeleteAsync().ConfigureAwait(false); } } From 36f6866a09b30b40987532c7790693bdabc6f27f Mon Sep 17 00:00:00 2001 From: Ionite Date: Fri, 17 Nov 2023 17:54:18 -0500 Subject: [PATCH 12/30] Support DefaultUnknownEnumConverter as dict keys --- .../Json/DefaultUnknownEnumConverter.cs | 62 ++++++++++++++++--- 1 file changed, 55 insertions(+), 7 deletions(-) diff --git a/StabilityMatrix.Core/Converters/Json/DefaultUnknownEnumConverter.cs b/StabilityMatrix.Core/Converters/Json/DefaultUnknownEnumConverter.cs index 62faa835..2de1f45d 100644 --- a/StabilityMatrix.Core/Converters/Json/DefaultUnknownEnumConverter.cs +++ b/StabilityMatrix.Core/Converters/Json/DefaultUnknownEnumConverter.cs @@ -4,11 +4,16 @@ using StabilityMatrix.Core.Extensions; namespace StabilityMatrix.Core.Converters.Json; -public class DefaultUnknownEnumConverter : JsonConverter where T : Enum +public class DefaultUnknownEnumConverter : JsonConverter + where T : Enum { - public override T Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + public override T Read( + ref Utf8JsonReader reader, + Type typeToConvert, + JsonSerializerOptions options + ) { - if (reader.TokenType != JsonTokenType.String) + if (reader.TokenType != JsonTokenType.String) { throw new JsonException(); } @@ -16,15 +21,15 @@ public class DefaultUnknownEnumConverter : JsonConverter where T : Enum var enumText = reader.GetString()?.Replace(" ", "_"); if (Enum.TryParse(typeof(T), enumText, true, out var result)) { - return (T) result!; + return (T)result!; } // Unknown value handling - if (Enum.TryParse(typeof(T), "Unknown", true, out var unknownResult)) + if (Enum.TryParse(typeof(T), "Unknown", true, out var unknownResult)) { - return (T) unknownResult!; + return (T)unknownResult!; } - + throw new JsonException($"Unable to parse '{enumText}' to enum '{typeof(T)}'."); } @@ -38,4 +43,47 @@ public class DefaultUnknownEnumConverter : JsonConverter where T : Enum writer.WriteStringValue(value.GetStringValue().Replace("_", " ")); } + + /// + public override T ReadAsPropertyName( + ref Utf8JsonReader reader, + Type typeToConvert, + JsonSerializerOptions options + ) + { + if (reader.TokenType != JsonTokenType.PropertyName) + { + throw new JsonException(); + } + + var enumText = reader.GetString()?.Replace(" ", "_"); + if (Enum.TryParse(typeof(T), enumText, true, out var result)) + { + return (T)result!; + } + + // Unknown value handling + if (Enum.TryParse(typeof(T), "Unknown", true, out var unknownResult)) + { + return (T)unknownResult!; + } + + throw new JsonException($"Unable to parse '{enumText}' to enum '{typeof(T)}'."); + } + + /// + public override void WriteAsPropertyName( + Utf8JsonWriter writer, + T? value, + JsonSerializerOptions options + ) + { + if (value == null) + { + writer.WriteNullValue(); + return; + } + + writer.WritePropertyName(value.GetStringValue().Replace("_", " ")); + } } From 78632fb14fd2c22d526f7d091b69891a40e9004e Mon Sep 17 00:00:00 2001 From: Ionite Date: Fri, 17 Nov 2023 18:00:56 -0500 Subject: [PATCH 13/30] Fix Delete method to have consistent not exist behavior with DeleteAsync --- StabilityMatrix.Core/Models/FileInterfaces/DirectoryPath.cs | 2 +- StabilityMatrix.Core/Updater/UpdateHelper.cs | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/StabilityMatrix.Core/Models/FileInterfaces/DirectoryPath.cs b/StabilityMatrix.Core/Models/FileInterfaces/DirectoryPath.cs index a996faf6..bd94d5f8 100644 --- a/StabilityMatrix.Core/Models/FileInterfaces/DirectoryPath.cs +++ b/StabilityMatrix.Core/Models/FileInterfaces/DirectoryPath.cs @@ -106,7 +106,7 @@ public class DirectoryPath : FileSystemPath, IPathObject, IEnumerable /// Deletes the directory. /// - public void Delete() => Directory.Delete(FullPath); + public void Delete() => Info.Delete(); /// Deletes the directory asynchronously. public Task DeleteAsync() => Task.Run(Delete); diff --git a/StabilityMatrix.Core/Updater/UpdateHelper.cs b/StabilityMatrix.Core/Updater/UpdateHelper.cs index d67e9bfe..061c7de4 100644 --- a/StabilityMatrix.Core/Updater/UpdateHelper.cs +++ b/StabilityMatrix.Core/Updater/UpdateHelper.cs @@ -62,6 +62,7 @@ public class UpdateHelper : IUpdateHelper UpdateFolder.Info.Attributes |= FileAttributes.Hidden; var downloadFile = UpdateFolder.JoinFile(Path.GetFileName(updateInfo.Url.ToString())); + var extractDir = UpdateFolder.JoinDir("extract"); try @@ -79,7 +80,7 @@ public class UpdateHelper : IUpdateHelper // Unzip if needed if (downloadFile.Extension == ".zip") { - await extractDir.DeleteAsync().ConfigureAwait(false); + await extractDir.DeleteAsync(true).ConfigureAwait(false); extractDir.Create(); await ArchiveHelper @@ -103,7 +104,7 @@ public class UpdateHelper : IUpdateHelper // Clean up original download await downloadFile.DeleteAsync().ConfigureAwait(false); // Clean up extract dir - await extractDir.DeleteAsync().ConfigureAwait(false); + await extractDir.DeleteAsync(true).ConfigureAwait(false); } } From bbf5b3990ee09cc82355fc9d1cc417b8f56b1abe Mon Sep 17 00:00:00 2001 From: Ionite Date: Fri, 17 Nov 2023 19:05:13 -0500 Subject: [PATCH 14/30] Fix MoveToDirectoryAsync --- StabilityMatrix.Core/Models/FileInterfaces/FilePath.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/StabilityMatrix.Core/Models/FileInterfaces/FilePath.cs b/StabilityMatrix.Core/Models/FileInterfaces/FilePath.cs index cd0f6887..a083fba6 100644 --- a/StabilityMatrix.Core/Models/FileInterfaces/FilePath.cs +++ b/StabilityMatrix.Core/Models/FileInterfaces/FilePath.cs @@ -174,7 +174,7 @@ public partial class FilePath : FileSystemPath, IPathObject /// public async Task MoveToDirectoryAsync(DirectoryPath directory) { - await Task.Run(() => Info.MoveTo(directory.FullPath)).ConfigureAwait(false); + await Task.Run(() => Info.MoveTo(directory.JoinFile(Name), true)).ConfigureAwait(false); // Return the new path return directory.JoinFile(this); } From 8c9552d0b2c79621b3a70f7715e9a3808869ed83 Mon Sep 17 00:00:00 2001 From: Ionite Date: Fri, 17 Nov 2023 19:05:31 -0500 Subject: [PATCH 15/30] Fix deleting unpack dir not exist yet --- StabilityMatrix.Core/Updater/UpdateHelper.cs | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/StabilityMatrix.Core/Updater/UpdateHelper.cs b/StabilityMatrix.Core/Updater/UpdateHelper.cs index 061c7de4..14f7401e 100644 --- a/StabilityMatrix.Core/Updater/UpdateHelper.cs +++ b/StabilityMatrix.Core/Updater/UpdateHelper.cs @@ -80,7 +80,10 @@ public class UpdateHelper : IUpdateHelper // Unzip if needed if (downloadFile.Extension == ".zip") { - await extractDir.DeleteAsync(true).ConfigureAwait(false); + if (extractDir.Exists) + { + await extractDir.DeleteAsync(true).ConfigureAwait(false); + } extractDir.Create(); await ArchiveHelper @@ -104,7 +107,10 @@ public class UpdateHelper : IUpdateHelper // Clean up original download await downloadFile.DeleteAsync().ConfigureAwait(false); // Clean up extract dir - await extractDir.DeleteAsync(true).ConfigureAwait(false); + if (extractDir.Exists) + { + await extractDir.DeleteAsync(true).ConfigureAwait(false); + } } } From 6609d0fabcd377e3ca3cbf27d71c34097a52099c Mon Sep 17 00:00:00 2001 From: Ionite Date: Fri, 17 Nov 2023 19:09:50 -0500 Subject: [PATCH 16/30] Use indeterminate progress to be smoother --- StabilityMatrix.Core/Updater/UpdateHelper.cs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/StabilityMatrix.Core/Updater/UpdateHelper.cs b/StabilityMatrix.Core/Updater/UpdateHelper.cs index 14f7401e..8e497fa4 100644 --- a/StabilityMatrix.Core/Updater/UpdateHelper.cs +++ b/StabilityMatrix.Core/Updater/UpdateHelper.cs @@ -86,9 +86,8 @@ public class UpdateHelper : IUpdateHelper } extractDir.Create(); - await ArchiveHelper - .Extract(downloadFile, extractDir, progress) - .ConfigureAwait(false); + progress.Report(new ProgressReport(-1, "Extracting...", isIndeterminate: true)); + await ArchiveHelper.Extract(downloadFile, extractDir).ConfigureAwait(false); // Move all (possibly nested) loose files to the root UpdateFolder foreach (var file in extractDir.EnumerateFiles("*.*", SearchOption.AllDirectories)) From 29fdf55f7f7e29bda33b95d3fbd8aa85c1c937f2 Mon Sep 17 00:00:00 2001 From: Ionite Date: Fri, 17 Nov 2023 19:59:14 -0500 Subject: [PATCH 17/30] Fix update indeterminate progress --- .../ViewModels/Dialogs/UpdateViewModel.cs | 5 +++++ StabilityMatrix.Avalonia/Views/Dialogs/UpdateDialog.axaml | 1 + StabilityMatrix.Core/Updater/UpdateHelper.cs | 4 +++- 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/StabilityMatrix.Avalonia/ViewModels/Dialogs/UpdateViewModel.cs b/StabilityMatrix.Avalonia/ViewModels/Dialogs/UpdateViewModel.cs index d83426d4..1078eb02 100644 --- a/StabilityMatrix.Avalonia/ViewModels/Dialogs/UpdateViewModel.cs +++ b/StabilityMatrix.Avalonia/ViewModels/Dialogs/UpdateViewModel.cs @@ -47,6 +47,9 @@ public partial class UpdateViewModel : ContentDialogViewModelBase [ObservableProperty] private int progressValue; + [ObservableProperty] + private bool isProgressIndeterminate; + [ObservableProperty] private bool showProgressBar; @@ -223,11 +226,13 @@ public partial class UpdateViewModel : ContentDialogViewModelBase Resources.TextTemplate_UpdatingPackage, Resources.Label_StabilityMatrix ); + await updateHelper.DownloadUpdate( UpdateInfo, new Progress(report => { ProgressValue = Convert.ToInt32(report.Percentage); + IsProgressIndeterminate = report.IsIndeterminate; }) ); diff --git a/StabilityMatrix.Avalonia/Views/Dialogs/UpdateDialog.axaml b/StabilityMatrix.Avalonia/Views/Dialogs/UpdateDialog.axaml index 70de14b0..51e6f43c 100644 --- a/StabilityMatrix.Avalonia/Views/Dialogs/UpdateDialog.axaml +++ b/StabilityMatrix.Avalonia/Views/Dialogs/UpdateDialog.axaml @@ -66,6 +66,7 @@ diff --git a/StabilityMatrix.Core/Updater/UpdateHelper.cs b/StabilityMatrix.Core/Updater/UpdateHelper.cs index 8e497fa4..d6d798e9 100644 --- a/StabilityMatrix.Core/Updater/UpdateHelper.cs +++ b/StabilityMatrix.Core/Updater/UpdateHelper.cs @@ -86,7 +86,9 @@ public class UpdateHelper : IUpdateHelper } extractDir.Create(); - progress.Report(new ProgressReport(-1, "Extracting...", isIndeterminate: true)); + progress.Report( + new ProgressReport(-1, isIndeterminate: true, type: ProgressType.Extract) + ); await ArchiveHelper.Extract(downloadFile, extractDir).ConfigureAwait(false); // Move all (possibly nested) loose files to the root UpdateFolder From 14a9643f9c58516ec80e4b77bfa6ee3bdea40679 Mon Sep 17 00:00:00 2001 From: Ionite Date: Fri, 17 Nov 2023 19:59:32 -0500 Subject: [PATCH 18/30] Fix update not launching correct file when renamed --- StabilityMatrix.Core/Updater/UpdateHelper.cs | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/StabilityMatrix.Core/Updater/UpdateHelper.cs b/StabilityMatrix.Core/Updater/UpdateHelper.cs index d6d798e9..f38b7597 100644 --- a/StabilityMatrix.Core/Updater/UpdateHelper.cs +++ b/StabilityMatrix.Core/Updater/UpdateHelper.cs @@ -91,11 +91,12 @@ public class UpdateHelper : IUpdateHelper ); await ArchiveHelper.Extract(downloadFile, extractDir).ConfigureAwait(false); - // Move all (possibly nested) loose files to the root UpdateFolder - foreach (var file in extractDir.EnumerateFiles("*.*", SearchOption.AllDirectories)) - { - await file.MoveToDirectoryAsync(UpdateFolder).ConfigureAwait(false); - } + // Find binary and move it up to the root + var binaryFile = extractDir + .EnumerateFiles("*.*", SearchOption.AllDirectories) + .First(f => f.Extension.ToLowerInvariant() is ".exe" or ".appimage"); + + await binaryFile.MoveToAsync(ExecutablePath).ConfigureAwait(false); } // Otherwise just rename else From dbf7abb8a7d73b0c9a455c9b2cf1b8bbcd1a1355 Mon Sep 17 00:00:00 2001 From: Ionite Date: Fri, 17 Nov 2023 20:01:43 -0500 Subject: [PATCH 19/30] Report completed progress after unzip --- StabilityMatrix.Core/Updater/UpdateHelper.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/StabilityMatrix.Core/Updater/UpdateHelper.cs b/StabilityMatrix.Core/Updater/UpdateHelper.cs index f38b7597..7ca9586f 100644 --- a/StabilityMatrix.Core/Updater/UpdateHelper.cs +++ b/StabilityMatrix.Core/Updater/UpdateHelper.cs @@ -103,6 +103,8 @@ public class UpdateHelper : IUpdateHelper { downloadFile.Rename(ExecutablePath.Name); } + + progress.Report(new ProgressReport(1d)); } finally { From ba30fe2eb9119b59c45afa1acb9d6ae2318e7841 Mon Sep 17 00:00:00 2001 From: ionite34 Date: Sat, 18 Nov 2023 14:24:57 -0500 Subject: [PATCH 20/30] Add ProcessResult.EnsureSuccessExitCode and other properties --- .../Exceptions/ProcessException.cs | 17 ++++++++++++++--- .../Processes/ProcessResult.cs | 18 +++++++++++++++++- 2 files changed, 31 insertions(+), 4 deletions(-) diff --git a/StabilityMatrix.Core/Exceptions/ProcessException.cs b/StabilityMatrix.Core/Exceptions/ProcessException.cs index 6d6435a2..b5903b5f 100644 --- a/StabilityMatrix.Core/Exceptions/ProcessException.cs +++ b/StabilityMatrix.Core/Exceptions/ProcessException.cs @@ -1,11 +1,22 @@ -namespace StabilityMatrix.Core.Exceptions; +using StabilityMatrix.Core.Processes; + +namespace StabilityMatrix.Core.Exceptions; /// /// Exception that is thrown when a process fails. /// public class ProcessException : Exception { - public ProcessException(string message) : base(message) + public ProcessResult? ProcessResult { get; } + + public ProcessException(string message) + : base(message) { } + + public ProcessException(ProcessResult processResult) + : base( + $"Process {processResult.ProcessName} exited with code {processResult.ExitCode}. {{StdOut = {processResult.StandardOutput}, StdErr = {processResult.StandardError}}}" + ) { + ProcessResult = processResult; } -} \ No newline at end of file +} diff --git a/StabilityMatrix.Core/Processes/ProcessResult.cs b/StabilityMatrix.Core/Processes/ProcessResult.cs index 241181d5..9fca6171 100644 --- a/StabilityMatrix.Core/Processes/ProcessResult.cs +++ b/StabilityMatrix.Core/Processes/ProcessResult.cs @@ -1,8 +1,24 @@ -namespace StabilityMatrix.Core.Processes; +using StabilityMatrix.Core.Exceptions; + +namespace StabilityMatrix.Core.Processes; public readonly record struct ProcessResult { public required int ExitCode { get; init; } public string? StandardOutput { get; init; } public string? StandardError { get; init; } + + public string? ProcessName { get; init; } + + public TimeSpan Elapsed { get; init; } + + public bool IsSuccessExitCode => ExitCode == 0; + + public void EnsureSuccessExitCode() + { + if (!IsSuccessExitCode) + { + throw new ProcessException(this); + } + } } From 8905bc2497239ceb22b544ecbe66ee7171e8b5d6 Mon Sep 17 00:00:00 2001 From: ionite34 Date: Sat, 18 Nov 2023 14:26:21 -0500 Subject: [PATCH 21/30] Populate additional properties of processresult --- StabilityMatrix.Core/Processes/ProcessRunner.cs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/StabilityMatrix.Core/Processes/ProcessRunner.cs b/StabilityMatrix.Core/Processes/ProcessRunner.cs index 3a20cc60..df1fa036 100644 --- a/StabilityMatrix.Core/Processes/ProcessRunner.cs +++ b/StabilityMatrix.Core/Processes/ProcessRunner.cs @@ -208,7 +208,9 @@ public static class ProcessRunner { ExitCode = process.ExitCode, StandardOutput = stdout, - StandardError = stderr + StandardError = stderr, + ProcessName = process.MachineName, + Elapsed = process.ExitTime - process.StartTime }; } From f7f1c2c6ea28fef7ccaec6517144a747b573bf6b Mon Sep 17 00:00:00 2001 From: ionite34 Date: Sat, 18 Nov 2023 14:26:45 -0500 Subject: [PATCH 22/30] Update SetupPip and InstallPackage to use ProcessResult --- StabilityMatrix.Core/Python/PyRunner.cs | 47 ++++++++++++++----------- 1 file changed, 27 insertions(+), 20 deletions(-) diff --git a/StabilityMatrix.Core/Python/PyRunner.cs b/StabilityMatrix.Core/Python/PyRunner.cs index ea7268d6..b053a281 100644 --- a/StabilityMatrix.Core/Python/PyRunner.cs +++ b/StabilityMatrix.Core/Python/PyRunner.cs @@ -124,8 +124,11 @@ public class PyRunner : IPyRunner { throw new FileNotFoundException("get-pip not found", GetPipPath); } - var p = ProcessRunner.StartAnsiProcess(PythonExePath, "-m get-pip"); - await ProcessRunner.WaitForExitConditionAsync(p); + + var result = await ProcessRunner + .GetProcessResultAsync(PythonExePath, "-m get-pip") + .ConfigureAwait(false); + result.EnsureSuccessExitCode(); } /// @@ -137,8 +140,10 @@ public class PyRunner : IPyRunner { throw new FileNotFoundException("pip not found", PipExePath); } - var p = ProcessRunner.StartAnsiProcess(PipExePath, $"install {package}"); - await ProcessRunner.WaitForExitConditionAsync(p); + var result = await ProcessRunner + .GetProcessResultAsync(PipExePath, $"install {package}") + .ConfigureAwait(false); + result.EnsureSuccessExitCode(); } /// @@ -159,15 +164,16 @@ public class PyRunner : IPyRunner try { return await Task.Run( - () => - { - using (Py.GIL()) + () => { - return func(); - } - }, - cancelToken - ); + using (Py.GIL()) + { + return func(); + } + }, + cancelToken + ) + .ConfigureAwait(false); } finally { @@ -193,15 +199,16 @@ public class PyRunner : IPyRunner try { await Task.Run( - () => - { - using (Py.GIL()) + () => { - action(); - } - }, - cancelToken - ); + using (Py.GIL()) + { + action(); + } + }, + cancelToken + ) + .ConfigureAwait(false); } finally { From d3734d65168b09111daff69c3c02a223d09e1edd Mon Sep 17 00:00:00 2001 From: ionite34 Date: Sat, 18 Nov 2023 15:35:06 -0500 Subject: [PATCH 23/30] Add error if AnsiProcess used with WaitForExit --- StabilityMatrix.Core/Processes/ProcessRunner.cs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/StabilityMatrix.Core/Processes/ProcessRunner.cs b/StabilityMatrix.Core/Processes/ProcessRunner.cs index df1fa036..4eb56a0a 100644 --- a/StabilityMatrix.Core/Processes/ProcessRunner.cs +++ b/StabilityMatrix.Core/Processes/ProcessRunner.cs @@ -427,6 +427,14 @@ public static class ProcessRunner CancellationToken cancelToken = default ) { + if (process is AnsiProcess) + { + throw new ArgumentException( + $"{nameof(WaitForExitConditionAsync)} does not support AnsiProcess, which uses custom async data reading", + nameof(process) + ); + } + var stdout = new StringBuilder(); var stderr = new StringBuilder(); process.OutputDataReceived += (_, args) => stdout.Append(args.Data); From b051ef15e58abcdfb491d500ddac5e8391281059 Mon Sep 17 00:00:00 2001 From: ionite34 Date: Sat, 18 Nov 2023 15:35:42 -0500 Subject: [PATCH 24/30] New RunGit overloads for better errors, refactors --- .../Helpers/UnixPrerequisiteHelper.cs | 19 ++- .../Helpers/WindowsPrerequisiteHelper.cs | 29 +++- StabilityMatrix.Core/Helper/ArchiveHelper.cs | 156 +++++++++++------- .../Helper/IPrerequisiteHelper.cs | 12 +- .../Helper/PrerequisiteHelper.cs | 27 ++- .../Models/Packages/BaseGitPackage.cs | 74 ++++----- .../Models/Packages/VladAutomatic.cs | 30 ++-- 7 files changed, 206 insertions(+), 141 deletions(-) diff --git a/StabilityMatrix.Avalonia/Helpers/UnixPrerequisiteHelper.cs b/StabilityMatrix.Avalonia/Helpers/UnixPrerequisiteHelper.cs index 04cb8ffd..e31be0da 100644 --- a/StabilityMatrix.Avalonia/Helpers/UnixPrerequisiteHelper.cs +++ b/StabilityMatrix.Avalonia/Helpers/UnixPrerequisiteHelper.cs @@ -126,16 +126,23 @@ public class UnixPrerequisiteHelper : IPrerequisiteHelper } } - public async Task RunGit( - string? workingDirectory = null, + /// + public Task RunGit( + ProcessArgs args, Action? onProcessOutput = null, - params string[] args + string? workingDirectory = null ) { - var command = - args.Length == 0 ? "git" : "git " + string.Join(" ", args.Select(ProcessRunner.Quote)); + // Async progress not supported on Unix + return RunGit(args, workingDirectory); + } + + /// + public async Task RunGit(ProcessArgs args, string? workingDirectory = null) + { + var command = args.Prepend("git"); - var result = await ProcessRunner.RunBashCommand(command, workingDirectory ?? ""); + var result = await ProcessRunner.RunBashCommand(command.ToArray(), workingDirectory ?? ""); if (result.ExitCode != 0) { Logger.Error( diff --git a/StabilityMatrix.Avalonia/Helpers/WindowsPrerequisiteHelper.cs b/StabilityMatrix.Avalonia/Helpers/WindowsPrerequisiteHelper.cs index 8df65b94..c6d007d0 100644 --- a/StabilityMatrix.Avalonia/Helpers/WindowsPrerequisiteHelper.cs +++ b/StabilityMatrix.Avalonia/Helpers/WindowsPrerequisiteHelper.cs @@ -6,6 +6,7 @@ using System.Threading.Tasks; using Microsoft.Win32; using NLog; using Octokit; +using StabilityMatrix.Core.Exceptions; using StabilityMatrix.Core.Helper; using StabilityMatrix.Core.Models.Progress; using StabilityMatrix.Core.Processes; @@ -64,23 +65,35 @@ public class WindowsPrerequisiteHelper : IPrerequisiteHelper } public async Task RunGit( - string? workingDirectory = null, - Action? onProcessOutput = null, - params string[] args + ProcessArgs args, + Action? onProcessOutput, + string? workingDirectory = null ) { var process = ProcessRunner.StartAnsiProcess( GitExePath, - args, - workingDirectory: workingDirectory, + args.ToArray(), + workingDirectory, + onProcessOutput, environmentVariables: new Dictionary { { "PATH", Compat.GetEnvPathWithExtensions(GitBinPath) } - }, - outputDataReceived: onProcessOutput + } ); + await process.WaitForExitAsync().ConfigureAwait(false); + if (process.ExitCode != 0) + { + throw new ProcessException($"Git exited with code {process.ExitCode}"); + } + } + + public async Task RunGit(ProcessArgs args, string? workingDirectory = null) + { + var result = await ProcessRunner + .GetProcessResultAsync(GitExePath, args, workingDirectory) + .ConfigureAwait(false); - await ProcessRunner.WaitForExitConditionAsync(process); + result.EnsureSuccessExitCode(); } public async Task GetGitOutput(string? workingDirectory = null, params string[] args) diff --git a/StabilityMatrix.Core/Helper/ArchiveHelper.cs b/StabilityMatrix.Core/Helper/ArchiveHelper.cs index f6f31ff8..8321f214 100644 --- a/StabilityMatrix.Core/Helper/ArchiveHelper.cs +++ b/StabilityMatrix.Core/Helper/ArchiveHelper.cs @@ -15,7 +15,6 @@ namespace StabilityMatrix.Core.Helper; public record struct ArchiveInfo(ulong Size, ulong CompressedSize); [SuppressMessage("ReSharper", "MemberCanBePrivate.Global")] - public static partial class ArchiveHelper { private static readonly Logger Logger = LogManager.GetCurrentClassLogger(); @@ -42,24 +41,24 @@ public static partial class ArchiveHelper throw new PlatformNotSupportedException("7z is not supported on this platform."); } } - + // HomeDir is set by ISettingsManager.TryFindLibrary() public static string HomeDir { get; set; } = string.Empty; public static string SevenZipPath => Path.Combine(HomeDir, "Assets", SevenZipFileName); - + [GeneratedRegex(@"(?<=Size:\s*)\d+|(?<=Compressed:\s*)\d+")] private static partial Regex Regex7ZOutput(); - + [GeneratedRegex(@"(?<=\s*)\d+(?=%)")] private static partial Regex Regex7ZProgressDigits(); - + [GeneratedRegex(@"(\d+)%.*- (.*)")] private static partial Regex Regex7ZProgressFull(); - + public static async Task TestArchive(string archivePath) { - var process = ProcessRunner.StartAnsiProcess(SevenZipPath, new[] {"t", archivePath}); + var process = ProcessRunner.StartAnsiProcess(SevenZipPath, new[] { "t", archivePath }); await process.WaitForExitAsync(); var output = await process.StandardOutput.ReadToEndAsync(); var matches = Regex7ZOutput().Matches(output); @@ -67,25 +66,29 @@ public static partial class ArchiveHelper var compressed = ulong.Parse(matches[1].Value); return new ArchiveInfo(size, compressed); } - + public static async Task AddToArchive7Z(string archivePath, string sourceDirectory) { // Start 7z in the parent directory of the source directory var sourceParent = Directory.GetParent(sourceDirectory)?.FullName ?? ""; // We must pass in as `directory\` for archive path to be correct var sourceDirName = new DirectoryInfo(sourceDirectory).Name; - var process = ProcessRunner.StartAnsiProcess(SevenZipPath, new[] - { - "a", archivePath, sourceDirName + @"\", "-y" - }, workingDirectory: sourceParent); - await ProcessRunner.WaitForExitConditionAsync(process); + + var result = await ProcessRunner + .GetProcessResultAsync( + SevenZipPath, + new[] { "a", archivePath, sourceDirName + @"\", "-y" }, + workingDirectory: sourceParent + ) + .ConfigureAwait(false); + result.EnsureSuccessExitCode(); } - + public static async Task Extract7Z(string archivePath, string extractDirectory) { var args = $"x {ProcessRunner.Quote(archivePath)} -o{ProcessRunner.Quote(extractDirectory)} -y"; - + Logger.Debug($"Starting process '{SevenZipPath}' with arguments '{args}'"); using var process = new Process(); @@ -99,7 +102,7 @@ public static partial class ArchiveHelper process.Start(); await ProcessRunner.WaitForExitConditionAsync(process); var output = await process.StandardOutput.ReadToEndAsync(); - + try { var matches = Regex7ZOutput().Matches(output); @@ -112,14 +115,19 @@ public static partial class ArchiveHelper throw new Exception($"Could not parse 7z output [{e.Message}]: {output.ToRepr()}"); } } - - public static async Task Extract7Z(string archivePath, string extractDirectory, IProgress progress) + + public static async Task Extract7Z( + string archivePath, + string extractDirectory, + IProgress progress + ) { var outputStore = new StringBuilder(); var onOutput = new Action(s => { - if (s == null) return; - + if (s == null) + return; + // Parse progress Logger.Trace($"7z: {s}"); outputStore.AppendLine(s); @@ -128,23 +136,30 @@ public static partial class ArchiveHelper { var percent = int.Parse(match.Groups[1].Value); var currentFile = match.Groups[2].Value; - progress.Report(new ProgressReport(percent / (float) 100, "Extracting", currentFile, type: ProgressType.Extract)); + progress.Report( + new ProgressReport( + percent / (float)100, + "Extracting", + currentFile, + type: ProgressType.Extract + ) + ); } }); progress.Report(new ProgressReport(-1, isIndeterminate: true, type: ProgressType.Extract)); - + // Need -bsp1 for progress reports var args = $"x {ProcessRunner.Quote(archivePath)} -o{ProcessRunner.Quote(extractDirectory)} -y -bsp1"; Logger.Debug($"Starting process '{SevenZipPath}' with arguments '{args}'"); - + var process = ProcessRunner.StartProcess(SevenZipPath, args, outputDataReceived: onOutput); await ProcessRunner.WaitForExitConditionAsync(process); - + progress.Report(new ProgressReport(1f, "Finished extracting", type: ProgressType.Extract)); - + var output = outputStore.ToString(); - + try { var matches = Regex7ZOutput().Matches(output); @@ -157,7 +172,7 @@ public static partial class ArchiveHelper throw new Exception($"Could not parse 7z output [{e.Message}]: {output.ToRepr()}"); } } - + /// /// Extracts a zipped tar (i.e. '.tar.gz') archive. /// First extracts the zipped tar, then extracts the tar and removes the tar. @@ -173,7 +188,7 @@ public static partial class ArchiveHelper } // Extract the tar.gz to tar await Extract7Z(archivePath, extractDirectory); - + // Extract the tar var tarPath = Path.Combine(extractDirectory, Path.GetFileNameWithoutExtension(archivePath)); if (!File.Exists(tarPath)) @@ -216,7 +231,11 @@ public static partial class ArchiveHelper /// /// /// Output directory, created if does not exist. - public static async Task Extract(string archivePath, string outputDirectory, IProgress? progress = default) + public static async Task Extract( + string archivePath, + string outputDirectory, + IProgress? progress = default + ) { Directory.CreateDirectory(outputDirectory); progress?.Report(new ProgressReport(-1, isIndeterminate: true)); @@ -229,48 +248,53 @@ public static partial class ArchiveHelper // If not available, use the size of the archive file if (total == 0) { - total = (ulong) new FileInfo(archivePath).Length; + total = (ulong)new FileInfo(archivePath).Length; } // Create an DispatchTimer that monitors the progress of the extraction - var progressMonitor = progress switch { + var progressMonitor = progress switch + { null => null, _ => new Timer(TimeSpan.FromMilliseconds(36)) }; - + if (progressMonitor != null) { progressMonitor.Elapsed += (_, _) => { - if (count == 0) return; + if (count == 0) + return; progress!.Report(new ProgressReport(count, total, message: "Extracting")); }; } - await Task.Factory.StartNew(() => - { - var extractOptions = new ExtractionOptions + await Task.Factory.StartNew( + () => { - Overwrite = true, - ExtractFullPath = true, - }; - using var stream = File.OpenRead(archivePath); - using var archive = ReaderFactory.Open(stream); + var extractOptions = new ExtractionOptions + { + Overwrite = true, + ExtractFullPath = true, + }; + using var stream = File.OpenRead(archivePath); + using var archive = ReaderFactory.Open(stream); - // Start the progress reporting timer - progressMonitor?.Start(); - - while (archive.MoveToNextEntry()) - { - var entry = archive.Entry; - if (!entry.IsDirectory) + // Start the progress reporting timer + progressMonitor?.Start(); + + while (archive.MoveToNextEntry()) { - count += (ulong) entry.CompressedSize; + var entry = archive.Entry; + if (!entry.IsDirectory) + { + count += (ulong)entry.CompressedSize; + } + archive.WriteEntryToDirectory(outputDirectory, extractOptions); } - archive.WriteEntryToDirectory(outputDirectory, extractOptions); - } - }, TaskCreationOptions.LongRunning); - + }, + TaskCreationOptions.LongRunning + ); + progress?.Report(new ProgressReport(progress: 1, message: "Done extracting")); progressMonitor?.Stop(); Logger.Info("Finished extracting archive {}", archivePath); @@ -285,7 +309,7 @@ public static partial class ArchiveHelper await using var stream = File.OpenRead(archivePath); await ExtractManaged(stream, outputDirectory); } - + /// /// Extract an archive to the output directory, using SharpCompress managed code. /// does not require 7z to be installed, but no progress reporting. @@ -298,7 +322,7 @@ public static partial class ArchiveHelper { var entry = reader.Entry; var outputPath = Path.Combine(outputDirectory, entry.Key); - + if (entry.IsDirectory) { if (!Directory.Exists(outputPath)) @@ -310,7 +334,7 @@ public static partial class ArchiveHelper { var folder = Path.GetDirectoryName(entry.Key)!; var destDir = Path.GetFullPath(Path.Combine(fullOutputDir, folder)); - + if (!Directory.Exists(destDir)) { if (!destDir.StartsWith(fullOutputDir, StringComparison.Ordinal)) @@ -322,20 +346,24 @@ public static partial class ArchiveHelper Directory.CreateDirectory(destDir); } - + // Check if symbolic link if (entry.LinkTarget != null) { // Not sure why but symlink entries have a key that ends with a space // and some broken path suffix, so we'll remove everything after the last space - Logger.Debug($"Checking if output path {outputPath} contains space char: {outputPath.Contains(' ')}"); + Logger.Debug( + $"Checking if output path {outputPath} contains space char: {outputPath.Contains(' ')}" + ); if (outputPath.Contains(' ')) { outputPath = outputPath[..outputPath.LastIndexOf(' ')]; } - - Logger.Debug($"Extracting symbolic link [{entry.Key.ToRepr()}] " + - $"({outputPath.ToRepr()} to {entry.LinkTarget.ToRepr()})"); + + Logger.Debug( + $"Extracting symbolic link [{entry.Key.ToRepr()}] " + + $"({outputPath.ToRepr()} to {entry.LinkTarget.ToRepr()})" + ); // Try to write link, if fail, continue copy file try { @@ -346,10 +374,12 @@ public static partial class ArchiveHelper } catch (IOException e) { - Logger.Warn($"Could not extract symbolic link, copying file instead: {e.Message}"); + Logger.Warn( + $"Could not extract symbolic link, copying file instead: {e.Message}" + ); } } - + // Write file await using var entryStream = reader.OpenEntryStream(); await using var fileStream = File.Create(outputPath); diff --git a/StabilityMatrix.Core/Helper/IPrerequisiteHelper.cs b/StabilityMatrix.Core/Helper/IPrerequisiteHelper.cs index 694d5990..6c937584 100644 --- a/StabilityMatrix.Core/Helper/IPrerequisiteHelper.cs +++ b/StabilityMatrix.Core/Helper/IPrerequisiteHelper.cs @@ -22,10 +22,16 @@ public interface IPrerequisiteHelper /// Run embedded git with the given arguments. /// Task RunGit( - string? workingDirectory = null, - Action? onProcessOutput = null, - params string[] args + ProcessArgs args, + Action? onProcessOutput, + string? workingDirectory = null ); + + /// + /// Run embedded git with the given arguments. + /// + Task RunGit(ProcessArgs args, string? workingDirectory = null); + Task GetGitOutput(string? workingDirectory = null, params string[] args); Task InstallTkinterIfNecessary(IProgress? progress = null); } diff --git a/StabilityMatrix.Core/Helper/PrerequisiteHelper.cs b/StabilityMatrix.Core/Helper/PrerequisiteHelper.cs index 19532a5b..de9898aa 100644 --- a/StabilityMatrix.Core/Helper/PrerequisiteHelper.cs +++ b/StabilityMatrix.Core/Helper/PrerequisiteHelper.cs @@ -1,8 +1,10 @@ -using System.Reflection; +using System.Diagnostics; +using System.Reflection; using System.Runtime.Versioning; using Microsoft.Extensions.Logging; using Microsoft.Win32; using Octokit; +using StabilityMatrix.Core.Exceptions; using StabilityMatrix.Core.Models.Progress; using StabilityMatrix.Core.Processes; using StabilityMatrix.Core.Services; @@ -61,18 +63,31 @@ public class PrerequisiteHelper : IPrerequisiteHelper } public async Task RunGit( - string? workingDirectory = null, - Action? onProcessOutput = null, - params string[] args + ProcessArgs args, + Action? onProcessOutput, + string? workingDirectory = null ) { var process = ProcessRunner.StartAnsiProcess( GitExePath, - args, + args.ToArray(), workingDirectory, onProcessOutput ); - await ProcessRunner.WaitForExitConditionAsync(process).ConfigureAwait(false); + await process.WaitForExitAsync().ConfigureAwait(false); + if (process.ExitCode != 0) + { + throw new ProcessException($"Git exited with code {process.ExitCode}"); + } + } + + public async Task RunGit(ProcessArgs args, string? workingDirectory = null) + { + var result = await ProcessRunner + .GetProcessResultAsync(GitExePath, args, workingDirectory) + .ConfigureAwait(false); + + result.EnsureSuccessExitCode(); } public async Task GetGitOutput(string? workingDirectory = null, params string[] args) diff --git a/StabilityMatrix.Core/Models/Packages/BaseGitPackage.cs b/StabilityMatrix.Core/Models/Packages/BaseGitPackage.cs index 84ed10df..2d4be812 100644 --- a/StabilityMatrix.Core/Models/Packages/BaseGitPackage.cs +++ b/StabilityMatrix.Core/Models/Packages/BaseGitPackage.cs @@ -185,13 +185,14 @@ public abstract class BaseGitPackage : BasePackage { await PrerequisiteHelper .RunGit( - null, - null, - "clone", - "--branch", - versionOptions.VersionTag, - GithubUrl, - $"\"{installLocation}\"" + new[] + { + "clone", + "--branch", + versionOptions.VersionTag, + GithubUrl, + installLocation + } ) .ConfigureAwait(false); } @@ -199,13 +200,14 @@ public abstract class BaseGitPackage : BasePackage { await PrerequisiteHelper .RunGit( - null, - null, - "clone", - "--branch", - versionOptions.BranchName, - GithubUrl, - $"\"{installLocation}\"" + new[] + { + "clone", + "--branch", + versionOptions.BranchName, + GithubUrl, + installLocation + } ) .ConfigureAwait(false); } @@ -213,7 +215,7 @@ public abstract class BaseGitPackage : BasePackage if (!versionOptions.IsLatest && !string.IsNullOrWhiteSpace(versionOptions.CommitHash)) { await PrerequisiteHelper - .RunGit(installLocation, null, "checkout", versionOptions.CommitHash) + .RunGit(new[] { "checkout", versionOptions.CommitHash }, installLocation) .ConfigureAwait(false); } @@ -327,12 +329,9 @@ public abstract class BaseGitPackage : BasePackage .ConfigureAwait(false); await PrerequisiteHelper .RunGit( - installedPackage.FullPath!, + new[] { "remote", "add", "origin", GithubUrl }, onConsoleOutput, - "remote", - "add", - "origin", - GithubUrl + installedPackage.FullPath ) .ConfigureAwait(false); } @@ -341,7 +340,7 @@ public abstract class BaseGitPackage : BasePackage { progress?.Report(new ProgressReport(-1f, "Fetching tags...", isIndeterminate: true)); await PrerequisiteHelper - .RunGit(installedPackage.FullPath!, onConsoleOutput, "fetch", "--tags") + .RunGit(new[] { "fetch", "--tags" }, onConsoleOutput, installedPackage.FullPath) .ConfigureAwait(false); progress?.Report( @@ -353,11 +352,9 @@ public abstract class BaseGitPackage : BasePackage ); await PrerequisiteHelper .RunGit( - installedPackage.FullPath!, + new[] { "checkout", versionOptions.VersionTag, "--force" }, onConsoleOutput, - "checkout", - versionOptions.VersionTag, - "--force" + installedPackage.FullPath ) .ConfigureAwait(false); @@ -381,7 +378,7 @@ public abstract class BaseGitPackage : BasePackage // fetch progress?.Report(new ProgressReport(-1f, "Fetching data...", isIndeterminate: true)); await PrerequisiteHelper - .RunGit(installedPackage.FullPath!, onConsoleOutput, "fetch") + .RunGit("fetch", onConsoleOutput, installedPackage.FullPath) .ConfigureAwait(false); if (versionOptions.IsLatest) @@ -396,11 +393,9 @@ public abstract class BaseGitPackage : BasePackage ); await PrerequisiteHelper .RunGit( - installedPackage.FullPath!, + new[] { "checkout", versionOptions.BranchName!, "--force" }, onConsoleOutput, - "checkout", - versionOptions.BranchName, - "--force" + installedPackage.FullPath ) .ConfigureAwait(false); @@ -408,12 +403,15 @@ public abstract class BaseGitPackage : BasePackage progress?.Report(new ProgressReport(-1f, "Pulling changes...", isIndeterminate: true)); await PrerequisiteHelper .RunGit( - installedPackage.FullPath!, + new[] + { + "pull", + "--autostash", + "origin", + installedPackage.Version.InstalledBranch! + }, onConsoleOutput, - "pull", - "--autostash", - "origin", - installedPackage.Version.InstalledBranch + installedPackage.FullPath! ) .ConfigureAwait(false); } @@ -429,11 +427,9 @@ public abstract class BaseGitPackage : BasePackage ); await PrerequisiteHelper .RunGit( - installedPackage.FullPath!, + new[] { "checkout", versionOptions.CommitHash!, "--force" }, onConsoleOutput, - "checkout", - versionOptions.CommitHash, - "--force" + installedPackage.FullPath ) .ConfigureAwait(false); } diff --git a/StabilityMatrix.Core/Models/Packages/VladAutomatic.cs b/StabilityMatrix.Core/Models/Packages/VladAutomatic.cs index 4810981d..6a5c6308 100644 --- a/StabilityMatrix.Core/Models/Packages/VladAutomatic.cs +++ b/StabilityMatrix.Core/Models/Packages/VladAutomatic.cs @@ -242,29 +242,28 @@ public class VladAutomatic : BaseGitPackage { await PrerequisiteHelper .RunGit( - installDir.Parent ?? "", - null, - "clone", - "https://github.com/vladmandic/automatic", - installDir.Name + new[] { "clone", "https://github.com/vladmandic/automatic", installDir.Name }, + installDir.Parent?.FullPath ?? "" ) .ConfigureAwait(false); await PrerequisiteHelper - .RunGit(installLocation, null, "checkout", downloadOptions.CommitHash) + .RunGit(new[] { "checkout", downloadOptions.CommitHash }, installLocation) .ConfigureAwait(false); } else if (!string.IsNullOrWhiteSpace(downloadOptions.BranchName)) { await PrerequisiteHelper .RunGit( - installDir.Parent ?? "", - null, - "clone", - "-b", - downloadOptions.BranchName, - "https://github.com/vladmandic/automatic", - installDir.Name + new[] + { + "clone", + "-b", + downloadOptions.BranchName, + "https://github.com/vladmandic/automatic", + installDir.Name + }, + installDir.Parent?.FullPath ?? "" ) .ConfigureAwait(false); } @@ -325,10 +324,9 @@ public class VladAutomatic : BaseGitPackage await PrerequisiteHelper .RunGit( - installedPackage.FullPath, + new[] { "checkout", versionOptions.BranchName! }, onConsoleOutput, - "checkout", - versionOptions.BranchName + installedPackage.FullPath ) .ConfigureAwait(false); From 5828aae3cb7b93b8fd3dafd62fd5d3b5d4eb0aa1 Mon Sep 17 00:00:00 2001 From: ionite34 Date: Sat, 18 Nov 2023 15:37:14 -0500 Subject: [PATCH 25/30] Update changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 28a6b5be..fa3ad42a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,8 @@ and this project adheres to [Semantic Versioning 2.0](https://semver.org/spec/v2 ## v2.6.2 ### Changed - Backend changes for auto-update schema v3, supporting customizable release channels and faster downloads with zip compression +### Fixed +- Better error reporting including outputs for git subprocess errors during package install / update ## v2.6.1 ### Changed From 0ba3b82308667e82492c961cb2ccd7f8b0298299 Mon Sep 17 00:00:00 2001 From: JT Date: Sun, 19 Nov 2023 22:29:59 -0800 Subject: [PATCH 26/30] Fixes a couple kohya_ss errors introduced in the latest version --- CHANGELOG.md | 3 + .../Models/Packages/KohyaSs.cs | 83 ++++++++++++++----- .../Models/Packages/VoltaML.cs | 2 +- 3 files changed, 68 insertions(+), 20 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 28a6b5be..af474bb1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,9 @@ and this project adheres to [Semantic Versioning 2.0](https://semver.org/spec/v2 ## v2.6.2 ### Changed - Backend changes for auto-update schema v3, supporting customizable release channels and faster downloads with zip compression +### Fixed +- Fixed `'accelerate' is not recognized as an internal or external command` error when starting training in kohya_ss +- Fixed some instances of `ModuleNotFoundError: No module named 'bitsandbytes.cuda_setup.paths'` error when using 8-bit optimizers in kohya_ss ## v2.6.1 ### Changed diff --git a/StabilityMatrix.Core/Models/Packages/KohyaSs.cs b/StabilityMatrix.Core/Models/Packages/KohyaSs.cs index 45a13d3d..99b2a6a3 100644 --- a/StabilityMatrix.Core/Models/Packages/KohyaSs.cs +++ b/StabilityMatrix.Core/Models/Packages/KohyaSs.cs @@ -1,4 +1,5 @@ using System.Text.RegularExpressions; +using Python.Runtime; using StabilityMatrix.Core.Attributes; using StabilityMatrix.Core.Extensions; using StabilityMatrix.Core.Helper; @@ -14,13 +15,19 @@ namespace StabilityMatrix.Core.Models.Packages; [Singleton(typeof(BasePackage))] public class KohyaSs : BaseGitPackage { + private readonly IPyRunner pyRunner; + public KohyaSs( IGithubApiCache githubApi, ISettingsManager settingsManager, IDownloadService downloadService, - IPrerequisiteHelper prerequisiteHelper + IPrerequisiteHelper prerequisiteHelper, + IPyRunner pyRunner ) - : base(githubApi, settingsManager, downloadService, prerequisiteHelper) { } + : base(githubApi, settingsManager, downloadService, prerequisiteHelper) + { + this.pyRunner = pyRunner; + } public override string Name => "kohya_ss"; public override string DisplayName { get; set; } = "kohya_ss"; @@ -147,6 +154,8 @@ public class KohyaSs : BaseGitPackage // Install venvRunner.RunDetached("setup/setup_sm.py", onConsoleOutput); await venvRunner.Process.WaitForExitAsync().ConfigureAwait(false); + + await venvRunner.PipInstall("bitsandbytes-windows").ConfigureAwait(false); } else if (Compat.IsLinux) { @@ -168,28 +177,64 @@ public class KohyaSs : BaseGitPackage await SetupVenv(installedPackagePath).ConfigureAwait(false); // update gui files to point to venv accelerate - var filesToUpdate = new[] + await pyRunner.RunInThreadWithLock(() => { - "lora_gui.py", - "dreambooth_gui.py", - "textual_inversion_gui.py", - Path.Combine("library", "wd14_caption_gui.py"), - "finetune_gui.py" - }; + var scope = Py.CreateScope(); + scope.Exec( + """ + import ast + + class StringReplacer(ast.NodeTransformer): + def __init__(self, old: str, new: str, replace_count: int = -1): + self.old = old + self.new = new + self.replace_count = replace_count + + def visit_Constant(self, node: ast.Constant) -> ast.Constant: + if isinstance(node.value, str) and self.old in node.value: + new_value = node.value.replace(self.old, self.new, self.replace_count) + node.value = new_value + return node + + def rewrite_module(self, module_text: str) -> str: + tree = ast.parse(module_text) + tree = self.visit(tree) + return ast.unparse(tree) + """ + ); - foreach (var file in filesToUpdate) - { - var path = Path.Combine(installedPackagePath, file); - var text = await File.ReadAllTextAsync(path).ConfigureAwait(false); var replacementAcceleratePath = Compat.IsWindows - ? @".\\venv\\scripts\\accelerate" + ? @".\venv\scripts\accelerate" : "./venv/bin/accelerate"; - text = text.Replace( - "run_cmd = f'accelerate launch", - $"run_cmd = f'{replacementAcceleratePath} launch" + + var replacer = scope.InvokeMethod( + "StringReplacer", + "accelerate".ToPython(), + $"{replacementAcceleratePath}".ToPython(), + 1.ToPython() ); - await File.WriteAllTextAsync(path, text).ConfigureAwait(false); - } + + var filesToUpdate = new[] + { + "lora_gui.py", + "dreambooth_gui.py", + "textual_inversion_gui.py", + Path.Combine("library", "wd14_caption_gui.py"), + "finetune_gui.py" + }; + + foreach (var file in filesToUpdate) + { + var path = Path.Combine(installedPackagePath, file); + var text = File.ReadAllText(path); + if (text.Contains(replacementAcceleratePath.Replace(@"\", @"\\"))) + continue; + + var result = replacer.InvokeMethod("rewrite_module", text.ToPython()); + var resultStr = result.ToString(); + File.WriteAllText(path, resultStr); + } + }); void HandleConsoleOutput(ProcessOutput s) { diff --git a/StabilityMatrix.Core/Models/Packages/VoltaML.cs b/StabilityMatrix.Core/Models/Packages/VoltaML.cs index eeef1ce3..933e4fb3 100644 --- a/StabilityMatrix.Core/Models/Packages/VoltaML.cs +++ b/StabilityMatrix.Core/Models/Packages/VoltaML.cs @@ -62,7 +62,7 @@ public class VoltaML : BaseGitPackage public override SharedFolderMethod RecommendedSharedFolderMethod => SharedFolderMethod.Symlink; public override IEnumerable AvailableTorchVersions => - new[] { TorchVersion.Cpu, TorchVersion.Cuda, TorchVersion.DirectMl, TorchVersion.Mps }; + new[] { TorchVersion.Cpu, TorchVersion.Cuda, TorchVersion.DirectMl }; public override IEnumerable AvailableSharedFolderMethods => new[] { SharedFolderMethod.Symlink, SharedFolderMethod.None }; From c8ed63f0170a7bf94aafe80b1d0f7db46c491090 Mon Sep 17 00:00:00 2001 From: JT Date: Sun, 19 Nov 2023 22:49:28 -0800 Subject: [PATCH 27/30] update patreon logo --- StabilityMatrix.Avalonia/Views/MainWindow.axaml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/StabilityMatrix.Avalonia/Views/MainWindow.axaml b/StabilityMatrix.Avalonia/Views/MainWindow.axaml index fb139dc7..ffd4bb7c 100644 --- a/StabilityMatrix.Avalonia/Views/MainWindow.axaml +++ b/StabilityMatrix.Avalonia/Views/MainWindow.axaml @@ -21,6 +21,15 @@ DockProperties.IsDropEnabled="True" x:Class="StabilityMatrix.Avalonia.Views.MainWindow"> + + + + + + + + + - + From 28716c10ef4b25d45062e5664cbcd0bf72c2fc6a Mon Sep 17 00:00:00 2001 From: JT Date: Sun, 19 Nov 2023 23:34:04 -0800 Subject: [PATCH 28/30] Delete failed installations if a new install is attempted in the same directory --- .../ViewModels/Dialogs/InstallerViewModel.cs | 13 ++++-- .../Dialogs/OneClickInstallViewModel.cs | 7 +++ .../Models/Packages/BaseGitPackage.cs | 44 ++++++------------- 3 files changed, 31 insertions(+), 33 deletions(-) diff --git a/StabilityMatrix.Avalonia/ViewModels/Dialogs/InstallerViewModel.cs b/StabilityMatrix.Avalonia/ViewModels/Dialogs/InstallerViewModel.cs index 55b40f30..df056b4d 100644 --- a/StabilityMatrix.Avalonia/ViewModels/Dialogs/InstallerViewModel.cs +++ b/StabilityMatrix.Avalonia/ViewModels/Dialogs/InstallerViewModel.cs @@ -16,6 +16,7 @@ using CommunityToolkit.Mvvm.Input; using FluentAvalonia.UI.Controls; using NLog; using StabilityMatrix.Avalonia.Controls; +using StabilityMatrix.Avalonia.Extensions; using StabilityMatrix.Avalonia.Languages; using StabilityMatrix.Avalonia.Services; using StabilityMatrix.Avalonia.ViewModels.Base; @@ -24,6 +25,7 @@ using StabilityMatrix.Core.Helper; using StabilityMatrix.Core.Helper.Factory; using StabilityMatrix.Core.Models; using StabilityMatrix.Core.Models.Database; +using StabilityMatrix.Core.Models.FileInterfaces; using StabilityMatrix.Core.Models.PackageModification; using StabilityMatrix.Core.Models.Packages; using StabilityMatrix.Core.Processes; @@ -221,7 +223,7 @@ public partial class InstallerViewModel : ContentDialogViewModelBase } } - private Task ActuallyInstall() + private async Task ActuallyInstall() { if (string.IsNullOrWhiteSpace(InstallName)) { @@ -232,12 +234,18 @@ public partial class InstallerViewModel : ContentDialogViewModelBase NotificationType.Error ) ); - return Task.CompletedTask; + return; } var setPackageInstallingStep = new SetPackageInstallingStep(settingsManager, InstallName); var installLocation = Path.Combine(settingsManager.LibraryDir, "Packages", InstallName); + if (Directory.Exists(installLocation)) + { + var installPath = new DirectoryPath(installLocation); + await installPath.DeleteVerboseAsync(); + } + var prereqStep = new SetupPrerequisitesStep(prerequisiteHelper, pyRunner); var downloadOptions = new DownloadPackageVersionOptions(); @@ -313,7 +321,6 @@ public partial class InstallerViewModel : ContentDialogViewModelBase }; Steps = steps; - return Task.CompletedTask; } public void Cancel() diff --git a/StabilityMatrix.Avalonia/ViewModels/Dialogs/OneClickInstallViewModel.cs b/StabilityMatrix.Avalonia/ViewModels/Dialogs/OneClickInstallViewModel.cs index 4c341aeb..8d1c033c 100644 --- a/StabilityMatrix.Avalonia/ViewModels/Dialogs/OneClickInstallViewModel.cs +++ b/StabilityMatrix.Avalonia/ViewModels/Dialogs/OneClickInstallViewModel.cs @@ -7,6 +7,7 @@ using System.Threading.Tasks; using CommunityToolkit.Mvvm.ComponentModel; using CommunityToolkit.Mvvm.Input; using Microsoft.Extensions.Logging; +using StabilityMatrix.Avalonia.Extensions; using StabilityMatrix.Avalonia.Languages; using StabilityMatrix.Avalonia.Services; using StabilityMatrix.Avalonia.ViewModels.Base; @@ -14,6 +15,7 @@ using StabilityMatrix.Core.Attributes; using StabilityMatrix.Core.Helper; using StabilityMatrix.Core.Helper.Factory; using StabilityMatrix.Core.Models; +using StabilityMatrix.Core.Models.FileInterfaces; using StabilityMatrix.Core.Models.PackageModification; using StabilityMatrix.Core.Models.Packages; using StabilityMatrix.Core.Python; @@ -139,6 +141,11 @@ public partial class OneClickInstallViewModel : ContentDialogViewModelBase "Packages", SelectedPackage.Name ); + if (Directory.Exists(installLocation)) + { + var installPath = new DirectoryPath(installLocation); + await installPath.DeleteVerboseAsync(); + } var downloadVersion = await SelectedPackage.GetLatestVersion(); var installedVersion = new InstalledPackageVersion { IsPrerelease = false }; diff --git a/StabilityMatrix.Core/Models/Packages/BaseGitPackage.cs b/StabilityMatrix.Core/Models/Packages/BaseGitPackage.cs index 2d4be812..e2686bb2 100644 --- a/StabilityMatrix.Core/Models/Packages/BaseGitPackage.cs +++ b/StabilityMatrix.Core/Models/Packages/BaseGitPackage.cs @@ -181,36 +181,20 @@ public abstract class BaseGitPackage : BasePackage IProgress? progress = null ) { - if (!string.IsNullOrWhiteSpace(versionOptions.VersionTag)) - { - await PrerequisiteHelper - .RunGit( - new[] - { - "clone", - "--branch", - versionOptions.VersionTag, - GithubUrl, - installLocation - } - ) - .ConfigureAwait(false); - } - else if (!string.IsNullOrWhiteSpace(versionOptions.BranchName)) - { - await PrerequisiteHelper - .RunGit( - new[] - { - "clone", - "--branch", - versionOptions.BranchName, - GithubUrl, - installLocation - } - ) - .ConfigureAwait(false); - } + await PrerequisiteHelper + .RunGit( + new[] + { + "clone", + "--branch", + !string.IsNullOrWhiteSpace(versionOptions.VersionTag) + ? versionOptions.VersionTag + : versionOptions.BranchName ?? MainBranch, + GithubUrl, + installLocation + } + ) + .ConfigureAwait(false); if (!versionOptions.IsLatest && !string.IsNullOrWhiteSpace(versionOptions.CommitHash)) { From dd0b17bf889ccfd89b6a8c982a74429e14c0c30a Mon Sep 17 00:00:00 2001 From: JT Date: Mon, 20 Nov 2023 15:21:24 -0800 Subject: [PATCH 29/30] fix bad crc on inference outputs --- CHANGELOG.md | 1 + StabilityMatrix.Avalonia/Helpers/PngDataHelper.cs | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 57647ef2..eb02670a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ and this project adheres to [Semantic Versioning 2.0](https://semver.org/spec/v2 - Better error reporting including outputs for git subprocess errors during package install / update - Fixed `'accelerate' is not recognized as an internal or external command` error when starting training in kohya_ss - Fixed some instances of `ModuleNotFoundError: No module named 'bitsandbytes.cuda_setup.paths'` error when using 8-bit optimizers in kohya_ss +- Fixed errors preventing Inference outputs from loading in the img2img tabs of other packages ## v2.6.1 ### Changed diff --git a/StabilityMatrix.Avalonia/Helpers/PngDataHelper.cs b/StabilityMatrix.Avalonia/Helpers/PngDataHelper.cs index 1d729cfe..f7d7e7c2 100644 --- a/StabilityMatrix.Avalonia/Helpers/PngDataHelper.cs +++ b/StabilityMatrix.Avalonia/Helpers/PngDataHelper.cs @@ -104,7 +104,7 @@ public static class PngDataHelper var dataBytes = Encoding.UTF8.GetBytes(textData); var textDataLength = BitConverter.GetBytes(dataBytes.Length).Reverse(); var textDataBytes = Text.Concat(dataBytes).ToArray(); - var crc = BitConverter.GetBytes(Crc32Algorithm.Compute(textDataBytes)); + var crc = BitConverter.GetBytes(Crc32Algorithm.Compute(textDataBytes)).Reverse(); return textDataLength.Concat(textDataBytes).Concat(crc).ToArray(); } From 9df0687fa34b74f9d21e4e0839affbc317a11463 Mon Sep 17 00:00:00 2001 From: JT Date: Mon, 20 Nov 2023 15:27:05 -0800 Subject: [PATCH 30/30] pass logger to deleteVerbose --- .../ViewModels/Dialogs/InstallerViewModel.cs | 21 ++++++++++--------- .../Dialogs/OneClickInstallViewModel.cs | 2 +- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/StabilityMatrix.Avalonia/ViewModels/Dialogs/InstallerViewModel.cs b/StabilityMatrix.Avalonia/ViewModels/Dialogs/InstallerViewModel.cs index df056b4d..c98d41bf 100644 --- a/StabilityMatrix.Avalonia/ViewModels/Dialogs/InstallerViewModel.cs +++ b/StabilityMatrix.Avalonia/ViewModels/Dialogs/InstallerViewModel.cs @@ -14,7 +14,7 @@ using Avalonia.Threading; using CommunityToolkit.Mvvm.ComponentModel; using CommunityToolkit.Mvvm.Input; using FluentAvalonia.UI.Controls; -using NLog; +using Microsoft.Extensions.Logging; using StabilityMatrix.Avalonia.Controls; using StabilityMatrix.Avalonia.Extensions; using StabilityMatrix.Avalonia.Languages; @@ -38,14 +38,13 @@ namespace StabilityMatrix.Avalonia.ViewModels.Dialogs; [Transient] public partial class InstallerViewModel : ContentDialogViewModelBase { - private static readonly Logger Logger = LogManager.GetCurrentClassLogger(); - private readonly ISettingsManager settingsManager; private readonly IPackageFactory packageFactory; private readonly IPyRunner pyRunner; private readonly IDownloadService downloadService; private readonly INotificationService notificationService; private readonly IPrerequisiteHelper prerequisiteHelper; + private readonly ILogger logger; [ObservableProperty] private BasePackage selectedPackage; @@ -132,7 +131,8 @@ public partial class InstallerViewModel : ContentDialogViewModelBase IPyRunner pyRunner, IDownloadService downloadService, INotificationService notificationService, - IPrerequisiteHelper prerequisiteHelper + IPrerequisiteHelper prerequisiteHelper, + ILogger logger ) { this.settingsManager = settingsManager; @@ -141,6 +141,7 @@ public partial class InstallerViewModel : ContentDialogViewModelBase this.downloadService = downloadService; this.notificationService = notificationService; this.prerequisiteHelper = prerequisiteHelper; + this.logger = logger; var filtered = packageFactory.GetAllAvailablePackages().Where(p => p.IsCompatible).ToList(); @@ -189,7 +190,7 @@ public partial class InstallerViewModel : ContentDialogViewModelBase } catch (Exception e) { - Logger.Warn("Error getting versions: {Exception}", e.ToString()); + logger.LogWarning("Error getting versions: {Exception}", e.ToString()); } finally { @@ -211,7 +212,7 @@ public partial class InstallerViewModel : ContentDialogViewModelBase else { var ex = result.Exception!; - Logger.Error(ex, $"Error installing package: {ex}"); + logger.LogError(ex, $"Error installing package: {ex}"); var dialog = new BetterContentDialog { @@ -243,7 +244,7 @@ public partial class InstallerViewModel : ContentDialogViewModelBase if (Directory.Exists(installLocation)) { var installPath = new DirectoryPath(installLocation); - await installPath.DeleteVerboseAsync(); + await installPath.DeleteVerboseAsync(logger); } var prereqStep = new SetupPrerequisitesStep(prerequisiteHelper, pyRunner); @@ -408,7 +409,7 @@ public partial class InstallerViewModel : ContentDialogViewModelBase Dispatcher.UIThread .InvokeAsync(async () => { - Logger.Debug($"Release mode: {IsReleaseMode}"); + logger.LogDebug($"Release mode: {IsReleaseMode}"); var versionOptions = await SelectedPackage.GetAllVersionOptions(); AvailableVersions = IsReleaseMode @@ -420,7 +421,7 @@ public partial class InstallerViewModel : ContentDialogViewModelBase return; ReleaseNotes = SelectedVersion.ReleaseNotesMarkdown; - Logger.Debug($"Loaded release notes for {ReleaseNotes}"); + logger.LogDebug($"Loaded release notes for {ReleaseNotes}"); if (!IsReleaseMode) { @@ -499,7 +500,7 @@ public partial class InstallerViewModel : ContentDialogViewModelBase } catch (Exception e) { - Logger.Warn($"Error getting commits: {e.Message}"); + logger.LogWarning(e, $"Error getting commits: {e.Message}"); } }) .SafeFireAndForget(); diff --git a/StabilityMatrix.Avalonia/ViewModels/Dialogs/OneClickInstallViewModel.cs b/StabilityMatrix.Avalonia/ViewModels/Dialogs/OneClickInstallViewModel.cs index 8d1c033c..8da42dd4 100644 --- a/StabilityMatrix.Avalonia/ViewModels/Dialogs/OneClickInstallViewModel.cs +++ b/StabilityMatrix.Avalonia/ViewModels/Dialogs/OneClickInstallViewModel.cs @@ -144,7 +144,7 @@ public partial class OneClickInstallViewModel : ContentDialogViewModelBase if (Directory.Exists(installLocation)) { var installPath = new DirectoryPath(installLocation); - await installPath.DeleteVerboseAsync(); + await installPath.DeleteVerboseAsync(logger); } var downloadVersion = await SelectedPackage.GetLatestVersion();