Merge pull request #422 from ionite34/fix-index

.config/.csharpierrc.json

@@ -1,4 +1,4 @@
 {
-    "printWidth": 120,
+    "printWidth": 110,
     "preprocessorSymbolSets": ["", "DEBUG", "DEBUG,CODE_STYLE"]
 }

.config/dotnet-tools.json

@@ -15,7 +15,7 @@
       ]
     },
     "csharpier": {
-      "version": "0.26.4",
+      "version": "0.26.7",
       "commands": [
         "dotnet-csharpier"
       ]

.csharpierrc.yaml

@@ -0,0 +1 @@
+printWidth: 110

CHANGELOG.md

@@ -12,6 +12,7 @@ and this project adheres to [Semantic Versioning 2.0](https://semver.org/spec/v2
 - Fixed denoise strength in Inference Text to Image
 - Fixed PathTooLongException for IPAdapter folders when using ComfyUI in Symlink mode
 - Fixed configs and symlinks not being cleaned up when switched to the opposite mode
+- Fixed model indexing stopping when encountering paths longer than 1021 bytes in length
 
 ## v2.7.3
 ### Added

StabilityMatrix.Core/Models/Database/LocalModelFile.cs

@@ -32,16 +32,19 @@ public class LocalModelFile
     /// <summary>
     /// File name of the relative path.
     /// </summary>
+    [BsonIgnore]
     public string FileName => Path.GetFileName(RelativePath);
 
     /// <summary>
     /// File name of the relative path without extension.
     /// </summary>
+    [BsonIgnore]
     public string FileNameWithoutExtension => Path.GetFileNameWithoutExtension(RelativePath);
 
     /// <summary>
     /// Relative file path from the shared folder type model directory.
     /// </summary>
+    [BsonIgnore]
     public string RelativePathFromSharedFolder => Path.GetRelativePath(SharedFolderType.GetStringValue(), RelativePath);
 
     public string GetFullPath(string rootModelDirectory)
@@ -54,8 +57,10 @@ public class LocalModelFile
         return PreviewImageRelativePath == null ? null : Path.Combine(rootModelDirectory, PreviewImageRelativePath);
     }
 
+    [BsonIgnore]
     public string FullPathGlobal => GetFullPath(GlobalConfig.LibraryDir.JoinDir("Models"));
 
+    [BsonIgnore]
     public string? PreviewImageFullPathGlobal => GetPreviewImageFullPath(GlobalConfig.LibraryDir.JoinDir("Models"));
 
     protected bool Equals(LocalModelFile other)

StabilityMatrix.Core/Services/ModelIndexService.cs

@@ -1,4 +1,5 @@
 using System.Diagnostics;
+using System.Text;
 using AsyncAwaitBestPractices;
 using Microsoft.Extensions.Logging;
 using StabilityMatrix.Core.Attributes;
@@ -18,8 +19,7 @@ public class ModelIndexService : IModelIndexService
     private readonly ILiteDbContext liteDbContext;
     private readonly ISettingsManager settingsManager;
 
-    public Dictionary<SharedFolderType, List<LocalModelFile>> ModelIndex { get; private set; } =
+    public Dictionary<SharedFolderType, List<LocalModelFile>> ModelIndex { get; private set; } = new();
-        new();
 
     public ModelIndexService(
         ILogger<ModelIndexService> logger,
@@ -48,8 +48,8 @@ public class ModelIndexService : IModelIndexService
     /// <inheritdoc />
     public async Task<IReadOnlyList<LocalModelFile>> GetModelsOfType(SharedFolderType type)
     {
-        return await liteDbContext.LocalModelFiles
+        return await liteDbContext
-            .Query()
+            .LocalModelFiles.Query()
             .Where(m => m.SharedFolderType == type)
             .ToArrayAsync()
             .ConfigureAwait(false);
@@ -87,8 +87,8 @@ public class ModelIndexService : IModelIndexService
 
         var newIndex = new Dictionary<SharedFolderType, List<LocalModelFile>>();
         foreach (
-            var file in modelsDir.Info
+            var file in modelsDir
-                .EnumerateFiles("*.*", SearchOption.AllDirectories)
+                .Info.EnumerateFiles("*.*", SearchOption.AllDirectories)
                 .Select(info => new FilePath(info))
         )
         {
@@ -111,10 +111,22 @@ public class ModelIndexService : IModelIndexService
                 continue;
             }
 
+            // Since RelativePath is the database key, for LiteDB this is limited to 1021 bytes
+            if (Encoding.UTF8.GetByteCount(relativePath) is var byteCount and > 1021)
+            {
+                logger.LogWarning(
+                    "Skipping model {Path} because it's path is too long ({Length} bytes)",
+                    relativePath,
+                    byteCount
+                );
+
+                continue;
+            }
+
             var localModel = new LocalModelFile
             {
                 RelativePath = relativePath,
-                SharedFolderType = sharedFolderType,
+                SharedFolderType = sharedFolderType
             };
 
             // Try to find a connected model info
@@ -132,18 +144,15 @@ public class ModelIndexService : IModelIndexService
             }
 
             // Try to find a preview image
-            var previewImagePath = LocalModelFile.SupportedImageExtensions
+            var previewImagePath = LocalModelFile
-                .Select(
+                .SupportedImageExtensions.Select(
                     ext => file.Directory!.JoinFile($"{file.NameWithoutExtension}.preview{ext}")
                 )
                 .FirstOrDefault(path => path.Exists);
 
             if (previewImagePath != null)
             {
-                localModel.PreviewImageRelativePath = Path.GetRelativePath(
+                localModel.PreviewImageRelativePath = Path.GetRelativePath(modelsDir, previewImagePath);
-                    modelsDir,
-                    previewImagePath
-                );
             }
 
             // Insert into database