Browse Source

PyVenv fixes and reformat

pull/55/head
Ionite 1 year ago
parent
commit
0a423640fe
No known key found for this signature in database
  1. 99
      StabilityMatrix.Core/Python/PyVenvRunner.cs

99
StabilityMatrix.Core/Python/PyVenvRunner.cs

@ -21,18 +21,18 @@ public class PyVenvRunner : IDisposable, IAsyncDisposable
public const string TorchPipInstallArgsCuda = public const string TorchPipInstallArgsCuda =
"torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/cu118"; "torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/cu118";
public const string TorchPipInstallArgsCpu = public const string TorchPipInstallArgsCpu = "torch torchvision torchaudio";
"torch torchvision torchaudio"; public const string TorchPipInstallArgsDirectML = "torch-directml";
public const string TorchPipInstallArgsDirectML =
"torch-directml";
/// <summary> /// <summary>
/// Relative path to the site-packages folder from the venv root. /// Relative path to the site-packages folder from the venv root.
/// This is platform specific. /// This is platform specific.
/// </summary> /// </summary>
public static string RelativeSitePackagesPath => Compat.Switch( public static string RelativeSitePackagesPath =>
Compat.Switch(
(PlatformKind.Windows, "Lib/site-packages"), (PlatformKind.Windows, "Lib/site-packages"),
(PlatformKind.Unix, "lib/python3.10/site-packages")); (PlatformKind.Unix, "lib/python3.10/site-packages")
);
/// <summary> /// <summary>
/// The process running the python executable. /// The process running the python executable.
@ -58,16 +58,17 @@ public class PyVenvRunner : IDisposable, IAsyncDisposable
/// Name of the python binary folder. /// Name of the python binary folder.
/// 'Scripts' on Windows, 'bin' on Unix. /// 'Scripts' on Windows, 'bin' on Unix.
/// </summary> /// </summary>
public static string RelativeBinPath => Compat.Switch( public static string RelativeBinPath =>
(PlatformKind.Windows, "Scripts"), Compat.Switch((PlatformKind.Windows, "Scripts"), (PlatformKind.Unix, "bin"));
(PlatformKind.Unix, "bin"));
/// <summary> /// <summary>
/// The relative path to the python executable. /// The relative path to the python executable.
/// </summary> /// </summary>
public static string RelativePythonPath => Compat.Switch( public static string RelativePythonPath =>
Compat.Switch(
(PlatformKind.Windows, Path.Combine("Scripts", "python.exe")), (PlatformKind.Windows, Path.Combine("Scripts", "python.exe")),
(PlatformKind.Unix, Path.Combine("bin", "python3"))); (PlatformKind.Unix, Path.Combine("bin", "python3"))
);
/// <summary> /// <summary>
/// The full path to the python executable. /// The full path to the python executable.
@ -77,9 +78,11 @@ public class PyVenvRunner : IDisposable, IAsyncDisposable
/// <summary> /// <summary>
/// The relative path to the pip executable. /// The relative path to the pip executable.
/// </summary> /// </summary>
public static string RelativePipPath => Compat.Switch( public static string RelativePipPath =>
Compat.Switch(
(PlatformKind.Windows, Path.Combine("Scripts", "pip.exe")), (PlatformKind.Windows, Path.Combine("Scripts", "pip.exe")),
(PlatformKind.Unix, Path.Combine("bin", "pip3"))); (PlatformKind.Unix, Path.Combine("bin", "pip3"))
);
/// <summary> /// <summary>
/// The full path to the pip executable. /// The full path to the pip executable.
@ -115,8 +118,13 @@ public class PyVenvRunner : IDisposable, IAsyncDisposable
RootPath.Create(); RootPath.Create();
// Create venv (copy mode if windows) // Create venv (copy mode if windows)
var args = new string[] { "-m", "virtualenv", var args = new string[]
Compat.IsWindows ? "--always-copy" : "", RootPath }; {
"-m",
"virtualenv",
Compat.IsWindows ? "--always-copy" : "",
RootPath
};
var venvProc = ProcessRunner.StartAnsiProcess(PyRunner.PythonExePath, args); var venvProc = ProcessRunner.StartAnsiProcess(PyRunner.PythonExePath, args);
await venvProc.WaitForExitAsync().ConfigureAwait(false); await venvProc.WaitForExitAsync().ConfigureAwait(false);
@ -126,7 +134,9 @@ public class PyVenvRunner : IDisposable, IAsyncDisposable
{ {
var output = await venvProc.StandardOutput.ReadToEndAsync().ConfigureAwait(false); var output = await venvProc.StandardOutput.ReadToEndAsync().ConfigureAwait(false);
output += await venvProc.StandardError.ReadToEndAsync().ConfigureAwait(false); output += await venvProc.StandardError.ReadToEndAsync().ConfigureAwait(false);
throw new InvalidOperationException($"Venv creation failed with code {returnCode}: {output}"); throw new InvalidOperationException(
$"Venv creation failed with code {returnCode}: {output}"
);
} }
} }
@ -137,7 +147,8 @@ public class PyVenvRunner : IDisposable, IAsyncDisposable
private void SetPyvenvCfg(string pythonDirectory) private void SetPyvenvCfg(string pythonDirectory)
{ {
// Skip if we are not created yet // Skip if we are not created yet
if (!Exists()) return; if (!Exists())
return;
// Path to pyvenv.cfg // Path to pyvenv.cfg
var cfgPath = Path.Combine(RootPath, "pyvenv.cfg"); var cfgPath = Path.Combine(RootPath, "pyvenv.cfg");
@ -158,8 +169,11 @@ public class PyVenvRunner : IDisposable, IAsyncDisposable
cfg.SetValue("top", "base-exec-prefix", pythonDirectory); cfg.SetValue("top", "base-exec-prefix", pythonDirectory);
cfg.SetValue("top", "base-executable", cfg.SetValue(
Path.Combine(pythonDirectory, Compat.IsWindows ? "python.exe" : RelativePythonPath)); "top",
"base-executable",
Path.Combine(pythonDirectory, Compat.IsWindows ? "python.exe" : RelativePythonPath)
);
// Convert to string for writing, strip the top section // Convert to string for writing, strip the top section
var cfgString = cfg.ToString()!.Replace(topSection, ""); var cfgString = cfg.ToString()!.Replace(topSection, "");
@ -180,7 +194,10 @@ public class PyVenvRunner : IDisposable, IAsyncDisposable
// Record output for errors // Record output for errors
var output = new StringBuilder(); var output = new StringBuilder();
var outputAction = outputDataReceived == null ? null : new Action<ProcessOutput>(s => var outputAction =
outputDataReceived == null
? null
: new Action<ProcessOutput>(s =>
{ {
Logger.Debug($"Pip output: {s.Text}"); Logger.Debug($"Pip output: {s.Text}");
// Record to output // Record to output
@ -197,7 +214,8 @@ public class PyVenvRunner : IDisposable, IAsyncDisposable
if (Process.ExitCode != 0) if (Process.ExitCode != 0)
{ {
throw new ProcessException( throw new ProcessException(
$"pip install failed with code {Process.ExitCode}: {output.ToString().ToRepr()}"); $"pip install failed with code {Process.ExitCode}: {output.ToString().ToRepr()}"
);
} }
} }
@ -212,14 +230,20 @@ public class PyVenvRunner : IDisposable, IAsyncDisposable
var outputAction = new Action<string?>(s => var outputAction = new Action<string?>(s =>
{ {
if (s == null) return; if (s == null)
return;
Logger.Debug("Pip output: {Text}", s); Logger.Debug("Pip output: {Text}", s);
output.Append(s); output.Append(s);
}); });
SetPyvenvCfg(PyRunner.PythonDir); SetPyvenvCfg(PyRunner.PythonDir);
using var process = ProcessRunner.StartProcess(PythonPath, arguments, using var process = ProcessRunner.StartProcess(
WorkingDirectory?.FullPath, outputAction, EnvironmentVariables); PythonPath,
arguments,
WorkingDirectory?.FullPath,
outputAction,
EnvironmentVariables
);
await process.WaitForExitAsync().ConfigureAwait(false); await process.WaitForExitAsync().ConfigureAwait(false);
return new ProcessResult return new ProcessResult
@ -234,17 +258,27 @@ public class PyVenvRunner : IDisposable, IAsyncDisposable
string arguments, string arguments,
Action<ProcessOutput>? outputDataReceived, Action<ProcessOutput>? outputDataReceived,
Action<int>? onExit = null, Action<int>? onExit = null,
bool unbuffered = true) bool unbuffered = true
)
{ {
if (!Exists()) if (!PythonPath.Exists)
{ {
throw new InvalidOperationException("Venv python process does not exist"); throw new FileNotFoundException("Venv python not found", PythonPath);
} }
SetPyvenvCfg(PyRunner.PythonDir); SetPyvenvCfg(PyRunner.PythonDir);
Logger.Debug($"Launching RunDetached at {PythonPath} with args {arguments}"); Logger.Info(
"Launching venv process [{PythonPath}] " +
"in working directory [{WorkingDirectory}] with args {arguments.ToRepr()}",
PythonPath,
WorkingDirectory,
arguments.ToRepr()
);
var filteredOutput = outputDataReceived == null ? null : new Action<ProcessOutput>(s => var filteredOutput =
outputDataReceived == null
? null
: new Action<ProcessOutput>(s =>
{ {
if (SuppressOutput.Any(s.Text.Contains)) if (SuppressOutput.Any(s.Text.Contains))
{ {
@ -286,10 +320,13 @@ public class PyVenvRunner : IDisposable, IAsyncDisposable
} }
} }
Process = ProcessRunner.StartAnsiProcess(PythonPath, arguments, Process = ProcessRunner.StartAnsiProcess(
PythonPath,
arguments,
workingDirectory: WorkingDirectory?.FullPath, workingDirectory: WorkingDirectory?.FullPath,
outputDataReceived: filteredOutput, outputDataReceived: filteredOutput,
environmentVariables: env); environmentVariables: env
);
if (onExit != null) if (onExit != null)
{ {

Loading…
Cancel
Save