KenSchae
/
StabilityMatrix
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity
Browse Source

Fix enumerator errors during unexpected EOF tokenization

pull/165/head
Ionite 1 year ago
parent
f6827e8a63
commit
b73dc6e7ec
No known key found for this signature in database
3 changed files with 80 additions and 61 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 28
      StabilityMatrix.Avalonia/Assets/ImagePrompt.tmLanguage.json
  2. 4
      StabilityMatrix.Avalonia/DialogHelper.cs
  3. 109
      StabilityMatrix.Avalonia/Models/Inference/Prompt.cs

28
StabilityMatrix.Avalonia/Assets/ImagePrompt.tmLanguage.json
Unescape View File

@ -175,10 +175,10 @@
} }
}, },
{ {
"include": "#colon" "include": "#comment"
}, },
{ {
"include": "#number" "include": "#escape"
}, },
{ {
"match": "[^\\s\\>]+", "match": "[^\\s\\>]+",
@ -214,6 +214,23 @@
"match": "[^,:\\[\\]\\(\\)\\<\\> \\\\]+", "match": "[^,:\\[\\]\\(\\)\\<\\> \\\\]+",
"name": "meta.embedded" "name": "meta.embedded"
}, },
"invalid_reserved" : {
"name": "invalid.illegal.reserved.prompt",
"patterns": [
{
"match": ":",
"name": "invalid.illegal.reserved.prompt"
},
{
"match": "\\)",
"name": "invalid.illegal.mismatched.parenthesis.closing.prompt"
},
{
"match": "\\(",
"name": "invalid.illegal.mismatched.parenthesis.opening.prompt"
}
]
},
"value": { "value": {
"patterns": [ "patterns": [
{ {
@ -244,12 +261,7 @@
"include": "#text" "include": "#text"
}, },
{ {
"name": "invalid.illegal.mismatched.parenthesis.closing.prompt", "include": "#invalid_reserved"
"match": "\\)"
},
{
"name": "invalid.illegal.mismatched.parenthesis.opening.prompt",
"match": "\\("
} }
] ]
} }

4
StabilityMatrix.Avalonia/DialogHelper.cs
Unescape View File

@ -348,7 +348,7 @@ public static class DialogHelper
var textEditor = new TextEditor var textEditor = new TextEditor
{ {
IsReadOnly = true, IsReadOnly = true,
WordWrap = true, WordWrap = false,
IsEnabled = false, IsEnabled = false,
ShowLineNumbers = false, ShowLineNumbers = false,
FontFamily = "Cascadia Code,Consolas,Menlo,Monospace", FontFamily = "Cascadia Code,Consolas,Menlo,Monospace",
@ -383,6 +383,8 @@ public static class DialogHelper
} }
}; };
textEditor.ScrollToHorizontalOffset(errorLineEndOffset - errorLineOffset);
var dialog = new BetterContentDialog var dialog = new BetterContentDialog
{ {
Title = title, Title = title,

109
StabilityMatrix.Avalonia/Models/Inference/Prompt.cs
Unescape View File

@ -120,31 +120,33 @@ public record Prompt
while (tokens.MoveNext()) while (tokens.MoveNext())
{ {
var token = tokens.Current; var currentToken = tokens.Current;
// For any invalid syntax, throw // For any invalid syntax, throw
if (token.Scopes.Any(s => s.Contains("invalid.illegal"))) if (currentToken.Scopes.Any(s => s.Contains("invalid.illegal")))
{ {
// Generic // Generic
throw new PromptSyntaxError( throw new PromptSyntaxError(
"Invalid Token", "Invalid Token",
token.StartIndex, currentToken.StartIndex,
GetSafeEndIndex(token.EndIndex) GetSafeEndIndex(currentToken.EndIndex)
); );
} }
// Find start of network token, until then just add to output // Find start of network token, until then just add to output
if (!token.Scopes.Contains("punctuation.definition.network.begin.prompt")) if (!currentToken.Scopes.Contains("punctuation.definition.network.begin.prompt"))
{ {
// Comments - ignore // Comments - ignore
if (token.Scopes.Any(s => s.Contains("comment.line"))) if (currentToken.Scopes.Any(s => s.Contains("comment.line")))
{ {
continue; continue;
} }
// Normal tags - Push to output // Normal tags - Push to output
outputTokens.Push(token); outputTokens.Push(currentToken);
outputText.Push(RawText[token.StartIndex..GetSafeEndIndex(token.EndIndex)]); outputText.Push(
RawText[currentToken.StartIndex..GetSafeEndIndex(currentToken.EndIndex)]
);
continue; continue;
} }
@ -152,22 +154,22 @@ public record Prompt
if (!tokens.MoveNext()) if (!tokens.MoveNext())
{ {
throw PromptSyntaxError.UnexpectedEndOfText( throw PromptSyntaxError.UnexpectedEndOfText(
token.StartIndex, currentToken.StartIndex,
GetSafeEndIndex(token.EndIndex) GetSafeEndIndex(currentToken.EndIndex)
); );
} }
var networkTypeToken = tokens.Current; currentToken = tokens.Current;
if (!networkTypeToken.Scopes.Contains("meta.embedded.network.type.prompt")) if (!currentToken.Scopes.Contains("meta.embedded.network.type.prompt"))
{ {
throw PromptSyntaxError.Network_ExpectedType( throw PromptSyntaxError.Network_ExpectedType(
networkTypeToken.StartIndex, currentToken.StartIndex,
GetSafeEndIndex(networkTypeToken.EndIndex) GetSafeEndIndex(currentToken.EndIndex)
); );
} }
var networkType = RawText[ var networkType = RawText[
networkTypeToken.StartIndex..GetSafeEndIndex(networkTypeToken.EndIndex) currentToken.StartIndex..GetSafeEndIndex(currentToken.EndIndex)
]; ];
// Match network type // Match network type
@ -178,8 +180,8 @@ public record Prompt
"embedding" => PromptExtraNetworkType.Embedding, "embedding" => PromptExtraNetworkType.Embedding,
_ _
=> throw PromptValidationError.Network_UnknownType( => throw PromptValidationError.Network_UnknownType(
networkTypeToken.StartIndex, currentToken.StartIndex,
GetSafeEndIndex(networkTypeToken.EndIndex) GetSafeEndIndex(currentToken.EndIndex)
) )
}; };
@ -187,16 +189,18 @@ public record Prompt
if (!tokens.MoveNext()) if (!tokens.MoveNext())
{ {
throw PromptSyntaxError.UnexpectedEndOfText( throw PromptSyntaxError.UnexpectedEndOfText(
tokens.Current.StartIndex, currentToken.StartIndex,
GetSafeEndIndex(tokens.Current.EndIndex) GetSafeEndIndex(currentToken.EndIndex)
); );
} }
currentToken = tokens.Current;
// Ensure next token is colon // Ensure next token is colon
if (!tokens.Current.Scopes.Contains("punctuation.separator.variable.prompt")) if (!currentToken.Scopes.Contains("punctuation.separator.variable.prompt"))
{ {
throw PromptSyntaxError.Network_ExpectedSeparator( throw PromptSyntaxError.Network_ExpectedSeparator(
tokens.Current.StartIndex, currentToken.StartIndex,
GetSafeEndIndex(tokens.Current.EndIndex) GetSafeEndIndex(currentToken.EndIndex)
); );
} }
@ -204,22 +208,22 @@ public record Prompt
if (!tokens.MoveNext()) if (!tokens.MoveNext())
{ {
throw PromptSyntaxError.UnexpectedEndOfText( throw PromptSyntaxError.UnexpectedEndOfText(
tokens.Current.StartIndex, currentToken.StartIndex,
GetSafeEndIndex(tokens.Current.EndIndex) GetSafeEndIndex(currentToken.EndIndex)
); );
} }
currentToken = tokens.Current;
var modelNameToken = tokens.Current; if (!currentToken.Scopes.Contains("meta.embedded.network.model.prompt"))
if (!tokens.Current.Scopes.Contains("meta.embedded.network.model.prompt"))
{ {
throw PromptSyntaxError.Network_ExpectedName( throw PromptSyntaxError.Network_ExpectedName(
tokens.Current.StartIndex, currentToken.StartIndex,
GetSafeEndIndex(tokens.Current.EndIndex) GetSafeEndIndex(currentToken.EndIndex)
); );
} }
var modelName = RawText[ var modelName = RawText[
modelNameToken.StartIndex..GetSafeEndIndex(modelNameToken.EndIndex) currentToken.StartIndex..GetSafeEndIndex(currentToken.EndIndex)
]; ];
// If index service provided, validate model name // If index service provided, validate model name
@ -235,8 +239,8 @@ public record Prompt
{ {
throw PromptValidationError.Network_UnknownModel( throw PromptValidationError.Network_UnknownModel(
modelName, modelName,
modelNameToken.StartIndex, currentToken.StartIndex,
GetSafeEndIndex(modelNameToken.EndIndex) GetSafeEndIndex(currentToken.EndIndex)
); );
} }
} }
@ -245,13 +249,13 @@ public record Prompt
if (!tokens.MoveNext()) if (!tokens.MoveNext())
{ {
throw PromptSyntaxError.UnexpectedEndOfText( throw PromptSyntaxError.UnexpectedEndOfText(
tokens.Current.StartIndex, currentToken.StartIndex,
GetSafeEndIndex(tokens.Current.EndIndex) GetSafeEndIndex(currentToken.EndIndex)
); );
} }
// If its a ending token instead, we can end here // If its a ending token instead, we can end here
if (tokens.Current.Scopes.Contains("punctuation.definition.network.end.prompt")) if (currentToken.Scopes.Contains("punctuation.definition.network.end.prompt"))
{ {
// If last entry on stack is a separator, remove it // If last entry on stack is a separator, remove it
if ( if (
@ -270,11 +274,11 @@ public record Prompt
} }
// Ensure next token is colon // Ensure next token is colon
if (!tokens.Current.Scopes.Contains("punctuation.separator.variable.prompt")) if (!currentToken.Scopes.Contains("punctuation.separator.variable.prompt"))
{ {
throw PromptSyntaxError.Network_ExpectedSeparator( throw PromptSyntaxError.Network_ExpectedSeparator(
tokens.Current.StartIndex, currentToken.StartIndex,
GetSafeEndIndex(tokens.Current.EndIndex) GetSafeEndIndex(currentToken.EndIndex)
); );
} }
@ -282,30 +286,30 @@ public record Prompt
if (!tokens.MoveNext()) if (!tokens.MoveNext())
{ {
throw PromptSyntaxError.UnexpectedEndOfText( throw PromptSyntaxError.UnexpectedEndOfText(
tokens.Current.StartIndex, currentToken.StartIndex,
GetSafeEndIndex(tokens.Current.EndIndex) GetSafeEndIndex(currentToken.EndIndex)
); );
} }
currentToken = tokens.Current;
var modelWeightToken = tokens.Current; if (!currentToken.Scopes.Contains("constant.numeric"))
if (!tokens.Current.Scopes.Contains("constant.numeric"))
{ {
throw PromptSyntaxError.Network_ExpectedWeight( throw PromptSyntaxError.Network_ExpectedWeight(
tokens.Current.StartIndex, currentToken.StartIndex,
GetSafeEndIndex(tokens.Current.EndIndex) GetSafeEndIndex(currentToken.EndIndex)
); );
} }
var modelWeight = RawText[ var modelWeight = RawText[
modelWeightToken.StartIndex..GetSafeEndIndex(modelWeightToken.EndIndex) currentToken.StartIndex..GetSafeEndIndex(currentToken.EndIndex)
]; ];
// Convert to double // Convert to double
if (!double.TryParse(modelWeight, out var weight)) if (!double.TryParse(modelWeight, out var weight))
{ {
throw PromptValidationError.Network_InvalidWeight( throw PromptValidationError.Network_InvalidWeight(
modelWeightToken.StartIndex, currentToken.StartIndex,
GetSafeEndIndex(modelWeightToken.EndIndex) GetSafeEndIndex(currentToken.EndIndex)
); );
} }
@ -313,16 +317,17 @@ public record Prompt
if (!tokens.MoveNext()) if (!tokens.MoveNext())
{ {
throw PromptSyntaxError.UnexpectedEndOfText( throw PromptSyntaxError.UnexpectedEndOfText(
tokens.Current.StartIndex, currentToken.StartIndex,
GetSafeEndIndex(tokens.Current.EndIndex) GetSafeEndIndex(currentToken.EndIndex)
); );
} }
var endToken = tokens.Current; currentToken = tokens.Current;
if (!endToken.Scopes.Contains("punctuation.definition.network.end.prompt"))
if (!currentToken.Scopes.Contains("punctuation.definition.network.end.prompt"))
{ {
throw PromptSyntaxError.UnexpectedEndOfText( throw PromptSyntaxError.UnexpectedEndOfText(
endToken.StartIndex, currentToken.StartIndex,
GetSafeEndIndex(endToken.EndIndex) GetSafeEndIndex(currentToken.EndIndex)
); );
} }

Write Preview
Loading…
Cancel
Save