KenSchae
/
StabilityMatrix
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity
Browse Source

PyVenv fixes and reformat

pull/55/head
Ionite 1 year ago
parent
099b8faafa
commit
0a423640fe
No known key found for this signature in database
1 changed files with 117 additions and 80 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 197
      StabilityMatrix.Core/Python/PyVenvRunner.cs

197
StabilityMatrix.Core/Python/PyVenvRunner.cs
Unescape View File

@ -20,20 +20,20 @@ public class PyVenvRunner : IDisposable, IAsyncDisposable
private static readonly Logger Logger = LogManager.GetCurrentClassLogger();
public const string TorchPipInstallArgsCuda =
"torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/cu118";
public const string TorchPipInstallArgsCpu =
"torch torchvision torchaudio";
public const string TorchPipInstallArgsDirectML =
"torch-directml";
"torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/cu118";
public const string TorchPipInstallArgsCpu = "torch torchvision torchaudio";
public const string TorchPipInstallArgsDirectML = "torch-directml";
/// <summary>
/// Relative path to the site-packages folder from the venv root.
/// This is platform specific.
/// </summary>
public static string RelativeSitePackagesPath => Compat.Switch(
(PlatformKind.Windows, "Lib/site-packages"),
(PlatformKind.Unix, "lib/python3.10/site-packages"));
public static string RelativeSitePackagesPath =>
Compat.Switch(
(PlatformKind.Windows, "Lib/site-packages"),
(PlatformKind.Unix, "lib/python3.10/site-packages")
);
/// <summary>
/// The process running the python executable.
/// </summary>
@ -48,51 +48,54 @@ public class PyVenvRunner : IDisposable, IAsyncDisposable
/// Optional working directory for the python process.
/// </summary>
public DirectoryPath? WorkingDirectory { get; set; }
/// <summary>
/// Optional environment variables for the python process.
/// </summary>
public IReadOnlyDictionary<string, string>? EnvironmentVariables { get; set; }
/// <summary>
/// Name of the python binary folder.
/// 'Scripts' on Windows, 'bin' on Unix.
/// </summary>
public static string RelativeBinPath => Compat.Switch(
(PlatformKind.Windows, "Scripts"),
(PlatformKind.Unix, "bin"));
public static string RelativeBinPath =>
Compat.Switch((PlatformKind.Windows, "Scripts"), (PlatformKind.Unix, "bin"));
/// <summary>
/// The relative path to the python executable.
/// </summary>
public static string RelativePythonPath => Compat.Switch(
(PlatformKind.Windows, Path.Combine("Scripts", "python.exe")),
(PlatformKind.Unix, Path.Combine("bin", "python3")));
public static string RelativePythonPath =>
Compat.Switch(
(PlatformKind.Windows, Path.Combine("Scripts", "python.exe")),
(PlatformKind.Unix, Path.Combine("bin", "python3"))
);
/// <summary>
/// The full path to the python executable.
/// </summary>
public FilePath PythonPath => RootPath.JoinFile(RelativePythonPath);
/// <summary>
/// The relative path to the pip executable.
/// </summary>
public static string RelativePipPath => Compat.Switch(
(PlatformKind.Windows, Path.Combine("Scripts", "pip.exe")),
(PlatformKind.Unix, Path.Combine("bin", "pip3")));
public static string RelativePipPath =>
Compat.Switch(
(PlatformKind.Windows, Path.Combine("Scripts", "pip.exe")),
(PlatformKind.Unix, Path.Combine("bin", "pip3"))
);
/// <summary>
/// The full path to the pip executable.
/// </summary>
public FilePath PipPath => RootPath.JoinFile(RelativePipPath);
/// <summary>
/// List of substrings to suppress from the output.
/// When a line contains any of these substrings, it will not be forwarded to callbacks.
/// A corresponding Info log will be written instead.
/// </summary>
public List<string> SuppressOutput { get; } = new() { "fatal: not a git repository" };
public PyVenvRunner(DirectoryPath path)
{
RootPath = path;
@ -115,8 +118,13 @@ public class PyVenvRunner : IDisposable, IAsyncDisposable
RootPath.Create();
// Create venv (copy mode if windows)
var args = new string[] { "-m", "virtualenv",
Compat.IsWindows ? "--always-copy" : "", RootPath };
var args = new string[]
{
"-m",
"virtualenv",
Compat.IsWindows ? "--always-copy" : "",
RootPath
};
var venvProc = ProcessRunner.StartAnsiProcess(PyRunner.PythonExePath, args);
await venvProc.WaitForExitAsync().ConfigureAwait(false);
@ -126,7 +134,9 @@ public class PyVenvRunner : IDisposable, IAsyncDisposable
{
var output = await venvProc.StandardOutput.ReadToEndAsync().ConfigureAwait(false);
output += await venvProc.StandardError.ReadToEndAsync().ConfigureAwait(false);
throw new InvalidOperationException($"Venv creation failed with code {returnCode}: {output}");
throw new InvalidOperationException(
$"Venv creation failed with code {returnCode}: {output}"
);
}
}
@ -137,7 +147,8 @@ public class PyVenvRunner : IDisposable, IAsyncDisposable
private void SetPyvenvCfg(string pythonDirectory)
{
// Skip if we are not created yet
if (!Exists()) return;
if (!Exists())
return;
// Path to pyvenv.cfg
var cfgPath = Path.Combine(RootPath, "pyvenv.cfg");
@ -145,22 +156,25 @@ public class PyVenvRunner : IDisposable, IAsyncDisposable
{
throw new FileNotFoundException("pyvenv.cfg not found", cfgPath);
}
Logger.Info("Updating pyvenv.cfg with embedded Python directory {PyDir}", pythonDirectory);
// Insert a top section
var topSection = "[top]" + Environment.NewLine;
var cfg = new ConfigParser(topSection + File.ReadAllText(cfgPath));
// Need to set all path keys - home, base-prefix, base-exec-prefix, base-executable
cfg.SetValue("top", "home", pythonDirectory);
cfg.SetValue("top", "base-prefix", pythonDirectory);
cfg.SetValue("top", "base-exec-prefix", pythonDirectory);
cfg.SetValue("top", "base-executable",
Path.Combine(pythonDirectory, Compat.IsWindows ? "python.exe" : RelativePythonPath));
cfg.SetValue(
"top",
"base-executable",
Path.Combine(pythonDirectory, Compat.IsWindows ? "python.exe" : RelativePythonPath)
);
// Convert to string for writing, strip the top section
var cfgString = cfg.ToString()!.Replace(topSection, "");
File.WriteAllText(cfgPath, cfgString);
@ -176,31 +190,35 @@ public class PyVenvRunner : IDisposable, IAsyncDisposable
{
throw new FileNotFoundException("pip not found", PipPath);
}
// Record output for errors
var output = new StringBuilder();
var outputAction = outputDataReceived == null ? null : new Action<ProcessOutput>(s =>
{
Logger.Debug($"Pip output: {s.Text}");
// Record to output
output.Append(s.Text);
// Forward to callback
outputDataReceived(s);
});
var outputAction =
outputDataReceived == null
? null
: new Action<ProcessOutput>(s =>
{
Logger.Debug($"Pip output: {s.Text}");
// Record to output
output.Append(s.Text);
// Forward to callback
outputDataReceived(s);
});
SetPyvenvCfg(PyRunner.PythonDir);
RunDetached($"-m pip install {args}", outputAction);
await Process.WaitForExitAsync().ConfigureAwait(false);
// Check return code
if (Process.ExitCode != 0)
{
throw new ProcessException(
$"pip install failed with code {Process.ExitCode}: {output.ToString().ToRepr()}");
$"pip install failed with code {Process.ExitCode}: {output.ToString().ToRepr()}"
);
}
}
/// <summary>
/// Run a command using the venv Python executable and return the result.
/// </summary>
@ -209,17 +227,23 @@ public class PyVenvRunner : IDisposable, IAsyncDisposable
{
// Record output for errors
var output = new StringBuilder();
var outputAction = new Action<string?>(s =>
{
if (s == null) return;
if (s == null)
return;
Logger.Debug("Pip output: {Text}", s);
output.Append(s);
});
SetPyvenvCfg(PyRunner.PythonDir);
using var process = ProcessRunner.StartProcess(PythonPath, arguments,
WorkingDirectory?.FullPath, outputAction, EnvironmentVariables);
using var process = ProcessRunner.StartProcess(
PythonPath,
arguments,
WorkingDirectory?.FullPath,
outputAction,
EnvironmentVariables
);
await process.WaitForExitAsync().ConfigureAwait(false);
return new ProcessResult
@ -231,45 +255,55 @@ public class PyVenvRunner : IDisposable, IAsyncDisposable
[MemberNotNull(nameof(Process))]
public void RunDetached(
string arguments,
string arguments,
Action<ProcessOutput>? outputDataReceived,
Action<int>? onExit = null,
bool unbuffered = true)
bool unbuffered = true
)
{
if (!Exists())
if (!PythonPath.Exists)
{
throw new InvalidOperationException("Venv python process does not exist");
throw new FileNotFoundException("Venv python not found", PythonPath);
}
SetPyvenvCfg(PyRunner.PythonDir);
Logger.Debug($"Launching RunDetached at {PythonPath} with args {arguments}");
var filteredOutput = outputDataReceived == null ? null : new Action<ProcessOutput>(s =>
{
if (SuppressOutput.Any(s.Text.Contains))
{
Logger.Info("Filtered output: {S}", s);
return;
}
outputDataReceived.Invoke(s);
});
Logger.Info(
"Launching venv process [{PythonPath}] " +
"in working directory [{WorkingDirectory}] with args {arguments.ToRepr()}",
PythonPath,
WorkingDirectory,
arguments.ToRepr()
);
var filteredOutput =
outputDataReceived == null
? null
: new Action<ProcessOutput>(s =>
{
if (SuppressOutput.Any(s.Text.Contains))
{
Logger.Info("Filtered output: {S}", s);
return;
}
outputDataReceived.Invoke(s);
});
var env = new Dictionary<string, string>();
if (EnvironmentVariables != null)
{
env.Update(EnvironmentVariables);
}
// Disable pip caching - uses significant memory for large packages like torch
env["PIP_NO_CACHE_DIR"] = "true";
// On windows, add portable git
// On windows, add portable git
if (Compat.IsWindows)
{
var portableGit = GlobalConfig.LibraryDir.JoinDir("PortableGit", "bin");
env["PATH"] = Compat.GetEnvPathWithExtensions(portableGit);
}
if (unbuffered)
{
env["PYTHONUNBUFFERED"] = "1";
@ -286,10 +320,13 @@ public class PyVenvRunner : IDisposable, IAsyncDisposable
}
}
Process = ProcessRunner.StartAnsiProcess(PythonPath, arguments,
Process = ProcessRunner.StartAnsiProcess(
PythonPath,
arguments,
workingDirectory: WorkingDirectory?.FullPath,
outputDataReceived: filteredOutput,
environmentVariables: env);
environmentVariables: env
);
if (onExit != null)
{
@ -300,7 +337,7 @@ public class PyVenvRunner : IDisposable, IAsyncDisposable
};
}
}
/// <summary>
/// Get entry points for a package.
/// https://packaging.python.org/en/latest/specifications/entry-points/#entry-points
@ -314,7 +351,7 @@ public class PyVenvRunner : IDisposable, IAsyncDisposable
results = entry_points(group='console_scripts', name='{entryPointName}')
print(tuple(results)[0].value, end='')
""";
var result = await Run($"-c \"{code}\"").ConfigureAwait(false);
if (result.ExitCode == 0 && !string.IsNullOrWhiteSpace(result.StandardOutput))
{
@ -323,7 +360,7 @@ public class PyVenvRunner : IDisposable, IAsyncDisposable
return null;
}
/// <summary>
/// Kills the running process and cancels stream readers, does not wait for exit.
/// </summary>
@ -335,11 +372,11 @@ public class PyVenvRunner : IDisposable, IAsyncDisposable
Process.Kill();
Process.Dispose();
}
Process = null;
GC.SuppressFinalize(this);
}
/// <summary>
/// Kills the running process, waits for exit.
/// </summary>

Write Preview
Loading…
Cancel
Save