KenSchae
/
StabilityMatrix
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity
Browse Source

Fix enumerator errors during unexpected EOF tokenization

pull/165/head
Ionite 1 year ago
parent
f6827e8a63
commit
b73dc6e7ec
No known key found for this signature in database
3 changed files with 80 additions and 61 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 28
      StabilityMatrix.Avalonia/Assets/ImagePrompt.tmLanguage.json
  2. 4
      StabilityMatrix.Avalonia/DialogHelper.cs
  3. 109
      StabilityMatrix.Avalonia/Models/Inference/Prompt.cs

28
StabilityMatrix.Avalonia/Assets/ImagePrompt.tmLanguage.json
Unescape View File

@ -175,10 +175,10 @@
}
},
{
"include": "#colon"
"include": "#comment"
},
{
"include": "#number"
"include": "#escape"
},
{
"match": "[^\\s\\>]+",
@ -214,6 +214,23 @@
"match": "[^,:\\[\\]\\(\\)\\<\\> \\\\]+",
"name": "meta.embedded"
},
"invalid_reserved" : {
"name": "invalid.illegal.reserved.prompt",
"patterns": [
{
"match": ":",
"name": "invalid.illegal.reserved.prompt"
},
{
"match": "\\)",
"name": "invalid.illegal.mismatched.parenthesis.closing.prompt"
},
{
"match": "\\(",
"name": "invalid.illegal.mismatched.parenthesis.opening.prompt"
}
]
},
"value": {
"patterns": [
{
@ -244,12 +261,7 @@
"include": "#text"
},
{
"name": "invalid.illegal.mismatched.parenthesis.closing.prompt",
"match": "\\)"
},
{
"name": "invalid.illegal.mismatched.parenthesis.opening.prompt",
"match": "\\("
"include": "#invalid_reserved"
}
]
}

4
StabilityMatrix.Avalonia/DialogHelper.cs
Unescape View File

@ -348,7 +348,7 @@ public static class DialogHelper
var textEditor = new TextEditor
{
IsReadOnly = true,
WordWrap = true,
WordWrap = false,
IsEnabled = false,
ShowLineNumbers = false,
FontFamily = "Cascadia Code,Consolas,Menlo,Monospace",
@ -383,6 +383,8 @@ public static class DialogHelper
}
};
textEditor.ScrollToHorizontalOffset(errorLineEndOffset - errorLineOffset);
var dialog = new BetterContentDialog
{
Title = title,

109
StabilityMatrix.Avalonia/Models/Inference/Prompt.cs
Unescape View File

@ -120,31 +120,33 @@ public record Prompt
while (tokens.MoveNext())
{
var token = tokens.Current;
var currentToken = tokens.Current;
// For any invalid syntax, throw
if (token.Scopes.Any(s => s.Contains("invalid.illegal")))
if (currentToken.Scopes.Any(s => s.Contains("invalid.illegal")))
{
// Generic
throw new PromptSyntaxError(
"Invalid Token",
token.StartIndex,
GetSafeEndIndex(token.EndIndex)
currentToken.StartIndex,
GetSafeEndIndex(currentToken.EndIndex)
);
}
// Find start of network token, until then just add to output
if (!token.Scopes.Contains("punctuation.definition.network.begin.prompt"))
if (!currentToken.Scopes.Contains("punctuation.definition.network.begin.prompt"))
{
// Comments - ignore
if (token.Scopes.Any(s => s.Contains("comment.line")))
if (currentToken.Scopes.Any(s => s.Contains("comment.line")))
{
continue;
}
// Normal tags - Push to output
outputTokens.Push(token);
outputText.Push(RawText[token.StartIndex..GetSafeEndIndex(token.EndIndex)]);
outputTokens.Push(currentToken);
outputText.Push(
RawText[currentToken.StartIndex..GetSafeEndIndex(currentToken.EndIndex)]
);
continue;
}
@ -152,22 +154,22 @@ public record Prompt
if (!tokens.MoveNext())
{
throw PromptSyntaxError.UnexpectedEndOfText(
token.StartIndex,
GetSafeEndIndex(token.EndIndex)
currentToken.StartIndex,
GetSafeEndIndex(currentToken.EndIndex)
);
}
var networkTypeToken = tokens.Current;
currentToken = tokens.Current;
if (!networkTypeToken.Scopes.Contains("meta.embedded.network.type.prompt"))
if (!currentToken.Scopes.Contains("meta.embedded.network.type.prompt"))
{
throw PromptSyntaxError.Network_ExpectedType(
networkTypeToken.StartIndex,
GetSafeEndIndex(networkTypeToken.EndIndex)
currentToken.StartIndex,
GetSafeEndIndex(currentToken.EndIndex)
);
}
var networkType = RawText[
networkTypeToken.StartIndex..GetSafeEndIndex(networkTypeToken.EndIndex)
currentToken.StartIndex..GetSafeEndIndex(currentToken.EndIndex)
];
// Match network type
@ -178,8 +180,8 @@ public record Prompt
"embedding" => PromptExtraNetworkType.Embedding,
_
=> throw PromptValidationError.Network_UnknownType(
networkTypeToken.StartIndex,
GetSafeEndIndex(networkTypeToken.EndIndex)
currentToken.StartIndex,
GetSafeEndIndex(currentToken.EndIndex)
)
};
@ -187,16 +189,18 @@ public record Prompt
if (!tokens.MoveNext())
{
throw PromptSyntaxError.UnexpectedEndOfText(
tokens.Current.StartIndex,
GetSafeEndIndex(tokens.Current.EndIndex)
currentToken.StartIndex,
GetSafeEndIndex(currentToken.EndIndex)
);
}
currentToken = tokens.Current;
// Ensure next token is colon
if (!tokens.Current.Scopes.Contains("punctuation.separator.variable.prompt"))
if (!currentToken.Scopes.Contains("punctuation.separator.variable.prompt"))
{
throw PromptSyntaxError.Network_ExpectedSeparator(
tokens.Current.StartIndex,
GetSafeEndIndex(tokens.Current.EndIndex)
currentToken.StartIndex,
GetSafeEndIndex(currentToken.EndIndex)
);
}
@ -204,22 +208,22 @@ public record Prompt
if (!tokens.MoveNext())
{
throw PromptSyntaxError.UnexpectedEndOfText(
tokens.Current.StartIndex,
GetSafeEndIndex(tokens.Current.EndIndex)
currentToken.StartIndex,
GetSafeEndIndex(currentToken.EndIndex)
);
}
currentToken = tokens.Current;
var modelNameToken = tokens.Current;
if (!tokens.Current.Scopes.Contains("meta.embedded.network.model.prompt"))
if (!currentToken.Scopes.Contains("meta.embedded.network.model.prompt"))
{
throw PromptSyntaxError.Network_ExpectedName(
tokens.Current.StartIndex,
GetSafeEndIndex(tokens.Current.EndIndex)
currentToken.StartIndex,
GetSafeEndIndex(currentToken.EndIndex)
);
}
var modelName = RawText[
modelNameToken.StartIndex..GetSafeEndIndex(modelNameToken.EndIndex)
currentToken.StartIndex..GetSafeEndIndex(currentToken.EndIndex)
];
// If index service provided, validate model name
@ -235,8 +239,8 @@ public record Prompt
{
throw PromptValidationError.Network_UnknownModel(
modelName,
modelNameToken.StartIndex,
GetSafeEndIndex(modelNameToken.EndIndex)
currentToken.StartIndex,
GetSafeEndIndex(currentToken.EndIndex)
);
}
}
@ -245,13 +249,13 @@ public record Prompt
if (!tokens.MoveNext())
{
throw PromptSyntaxError.UnexpectedEndOfText(
tokens.Current.StartIndex,
GetSafeEndIndex(tokens.Current.EndIndex)
currentToken.StartIndex,
GetSafeEndIndex(currentToken.EndIndex)
);
}
// If its a ending token instead, we can end here
if (tokens.Current.Scopes.Contains("punctuation.definition.network.end.prompt"))
if (currentToken.Scopes.Contains("punctuation.definition.network.end.prompt"))
{
// If last entry on stack is a separator, remove it
if (
@ -270,11 +274,11 @@ public record Prompt
}
// Ensure next token is colon
if (!tokens.Current.Scopes.Contains("punctuation.separator.variable.prompt"))
if (!currentToken.Scopes.Contains("punctuation.separator.variable.prompt"))
{
throw PromptSyntaxError.Network_ExpectedSeparator(
tokens.Current.StartIndex,
GetSafeEndIndex(tokens.Current.EndIndex)
currentToken.StartIndex,
GetSafeEndIndex(currentToken.EndIndex)
);
}
@ -282,30 +286,30 @@ public record Prompt
if (!tokens.MoveNext())
{
throw PromptSyntaxError.UnexpectedEndOfText(
tokens.Current.StartIndex,
GetSafeEndIndex(tokens.Current.EndIndex)
currentToken.StartIndex,
GetSafeEndIndex(currentToken.EndIndex)
);
}
currentToken = tokens.Current;
var modelWeightToken = tokens.Current;
if (!tokens.Current.Scopes.Contains("constant.numeric"))
if (!currentToken.Scopes.Contains("constant.numeric"))
{
throw PromptSyntaxError.Network_ExpectedWeight(
tokens.Current.StartIndex,
GetSafeEndIndex(tokens.Current.EndIndex)
currentToken.StartIndex,
GetSafeEndIndex(currentToken.EndIndex)
);
}
var modelWeight = RawText[
modelWeightToken.StartIndex..GetSafeEndIndex(modelWeightToken.EndIndex)
currentToken.StartIndex..GetSafeEndIndex(currentToken.EndIndex)
];
// Convert to double
if (!double.TryParse(modelWeight, out var weight))
{
throw PromptValidationError.Network_InvalidWeight(
modelWeightToken.StartIndex,
GetSafeEndIndex(modelWeightToken.EndIndex)
currentToken.StartIndex,
GetSafeEndIndex(currentToken.EndIndex)
);
}
@ -313,16 +317,17 @@ public record Prompt
if (!tokens.MoveNext())
{
throw PromptSyntaxError.UnexpectedEndOfText(
tokens.Current.StartIndex,
GetSafeEndIndex(tokens.Current.EndIndex)
currentToken.StartIndex,
GetSafeEndIndex(currentToken.EndIndex)
);
}
var endToken = tokens.Current;
if (!endToken.Scopes.Contains("punctuation.definition.network.end.prompt"))
currentToken = tokens.Current;
if (!currentToken.Scopes.Contains("punctuation.definition.network.end.prompt"))
{
throw PromptSyntaxError.UnexpectedEndOfText(
endToken.StartIndex,
GetSafeEndIndex(endToken.EndIndex)
currentToken.StartIndex,
GetSafeEndIndex(currentToken.EndIndex)
);
}

Write Preview
Loading…
Cancel
Save