using System.Diagnostics.CodeAnalysis;
using System.Text;
using NLog;
using Salaros.Configuration;
using StabilityMatrix.Core.Exceptions;
using StabilityMatrix.Core.Extensions;
using StabilityMatrix.Core.Helper;
using StabilityMatrix.Core.Models;
using StabilityMatrix.Core.Models.FileInterfaces;
using StabilityMatrix.Core.Processes;
namespace StabilityMatrix.Core.Python;
///
/// Python runner using a subprocess, mainly for venv support.
///
[SuppressMessage("ReSharper", "MemberCanBePrivate.Global")]
public class PyVenvRunner : IDisposable, IAsyncDisposable
{
private static readonly Logger Logger = LogManager.GetCurrentClassLogger();
public const string TorchPipInstallArgsCuda =
"torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/cu118";
public const string TorchPipInstallArgsCpu =
"torch torchvision torchaudio";
public const string TorchPipInstallArgsDirectML =
"torch-directml";
///
/// Relative path to the site-packages folder from the venv root.
/// This is platform specific.
///
public static string RelativeSitePackagesPath => Compat.Switch(
(PlatformKind.Windows, "Lib/site-packages"),
(PlatformKind.Unix, "lib/python3.10/site-packages"));
///
/// The process running the python executable.
///
public AnsiProcess? Process { get; private set; }
///
/// The path to the venv root directory.
///
public DirectoryPath RootPath { get; }
///
/// Name of the python binary folder.
/// 'Scripts' on Windows, 'bin' on Unix.
///
public static string RelativeBinPath => Compat.Switch(
(PlatformKind.Windows, "Scripts"),
(PlatformKind.Unix, "bin"));
///
/// The relative path to the python executable.
///
public static string RelativePythonPath => Compat.Switch(
(PlatformKind.Windows, Path.Combine("Scripts", "python.exe")),
(PlatformKind.Unix, Path.Combine("bin", "python3")));
///
/// The full path to the python executable.
///
public FilePath PythonPath => RootPath.JoinFile(RelativePythonPath);
///
/// The relative path to the pip executable.
///
public static string RelativePipPath => Compat.Switch(
(PlatformKind.Windows, Path.Combine("Scripts", "pip.exe")),
(PlatformKind.Unix, Path.Combine("bin", "pip3")));
///
/// The full path to the pip executable.
///
public FilePath PipPath => RootPath.JoinFile(RelativePipPath);
///
/// List of substrings to suppress from the output.
/// When a line contains any of these substrings, it will not be forwarded to callbacks.
/// A corresponding Info log will be written instead.
///
public List SuppressOutput { get; } = new() { "fatal: not a git repository" };
public PyVenvRunner(DirectoryPath path)
{
RootPath = path;
}
/// True if the venv has a Scripts\python.exe file
public bool Exists() => PythonPath.Exists;
///
/// Creates a venv at the configured path.
///
public async Task Setup(bool existsOk = false)
{
if (!existsOk && Exists())
{
throw new InvalidOperationException("Venv already exists");
}
// Create RootPath if it doesn't exist
RootPath.Create();
// Create venv (copy mode if windows)
var args = new string[] { "-m", "virtualenv",
Compat.IsWindows ? "--always-copy" : "", RootPath };
var venvProc = ProcessRunner.StartAnsiProcess(PyRunner.PythonExePath, args);
await venvProc.WaitForExitAsync();
// Check return code
var returnCode = venvProc.ExitCode;
if (returnCode != 0)
{
var output = await venvProc.StandardOutput.ReadToEndAsync();
output += await venvProc.StandardError.ReadToEndAsync();
throw new InvalidOperationException($"Venv creation failed with code {returnCode}: {output}");
}
}
///
/// Set current python path to pyvenv.cfg
/// This should be called before using the venv, in case user moves the venv directory.
///
private void SetPyvenvCfg(string pythonDirectory)
{
// Skip if we are not created yet
if (!Exists()) return;
// Path to pyvenv.cfg
var cfgPath = Path.Combine(RootPath, "pyvenv.cfg");
if (!File.Exists(cfgPath))
{
throw new FileNotFoundException("pyvenv.cfg not found", cfgPath);
}
Logger.Info("Updating pyvenv.cfg with embedded Python directory {PyDir}", pythonDirectory);
// Insert a top section
var topSection = "[top]" + Environment.NewLine;
var cfg = new ConfigParser(topSection + File.ReadAllText(cfgPath));
// Need to set all path keys - home, base-prefix, base-exec-prefix, base-executable
cfg.SetValue("top", "home", pythonDirectory);
cfg.SetValue("top", "base-prefix", pythonDirectory);
cfg.SetValue("top", "base-exec-prefix", pythonDirectory);
cfg.SetValue("top", "base-executable",
Path.Combine(pythonDirectory, Compat.IsWindows ? "python.exe" : RelativePythonPath));
// Convert to string for writing, strip the top section
var cfgString = cfg.ToString()!.Replace(topSection, "");
File.WriteAllText(cfgPath, cfgString);
}
///
/// Run a pip install command. Waits for the process to exit.
/// workingDirectory defaults to RootPath.
///
public async Task PipInstall(string args, string? workingDirectory = null, Action? outputDataReceived = null)
{
if (!File.Exists(PipPath))
{
throw new FileNotFoundException("pip not found", PipPath);
}
// Record output for errors
var output = new StringBuilder();
var outputAction = outputDataReceived == null ? null : new Action(s =>
{
Logger.Debug($"Pip output: {s.Text}");
// Record to output
output.Append(s.Text);
// Forward to callback
outputDataReceived(s);
});
SetPyvenvCfg(PyRunner.PythonDir);
RunDetached($"-m pip install {args}", outputAction, workingDirectory: workingDirectory ?? RootPath);
await Process.WaitForExitAsync();
// Check return code
if (Process.ExitCode != 0)
{
throw new ProcessException(
$"pip install failed with code {Process.ExitCode}: {output.ToString().ToRepr()}");
}
}
[MemberNotNull(nameof(Process))]
public void RunDetached(
string arguments,
Action? outputDataReceived,
Action? onExit = null,
bool unbuffered = true,
string workingDirectory = "",
Dictionary? environmentVariables = null)
{
if (!Exists())
{
throw new InvalidOperationException("Venv python process does not exist");
}
SetPyvenvCfg(PyRunner.PythonDir);
Logger.Debug($"Launching RunDetached at {PythonPath} with args {arguments}");
var filteredOutput = outputDataReceived == null ? null : new Action(s =>
{
if (SuppressOutput.Any(s.Text.Contains))
{
Logger.Info("Filtered output: {S}", s);
return;
}
outputDataReceived.Invoke(s);
});
environmentVariables ??= new Dictionary();
// Disable pip caching - uses significant memory for large packages like torch
environmentVariables["PIP_NO_CACHE_DIR"] = "true";
// On windows, add portable git
if (Compat.IsWindows)
{
var portableGit = GlobalConfig.LibraryDir.JoinDir("PortableGit", "bin");
environmentVariables["PATH"] = Compat.GetEnvPathWithExtensions(portableGit);
}
if (unbuffered)
{
environmentVariables["PYTHONUNBUFFERED"] = "1";
// If arguments starts with -, it's a flag, insert `u` after it for unbuffered mode
if (arguments.StartsWith('-'))
{
arguments = arguments.Insert(1, "u");
}
// Otherwise insert -u at the beginning
else
{
arguments = "-u " + arguments;
}
}
Process = ProcessRunner.StartAnsiProcess(PythonPath, arguments,
workingDirectory: workingDirectory,
outputDataReceived: filteredOutput,
environmentVariables: environmentVariables);
if (onExit != null)
{
Process.EnableRaisingEvents = true;
Process.Exited += (sender, _) =>
{
onExit((sender as AnsiProcess)?.ExitCode ?? -1);
};
}
}
public void Dispose()
{
Process?.CancelStreamReaders();
Process?.Kill();
Process = null;
GC.SuppressFinalize(this);
}
public async ValueTask DisposeAsync()
{
if (Process is not null)
{
Process.Kill();
await Process.WaitForExitAsync().ConfigureAwait(false);
}
Process = null;
GC.SuppressFinalize(this);
}
}