using System.Diagnostics.CodeAnalysis; using System.Text; using NLog; using Salaros.Configuration; using StabilityMatrix.Core.Exceptions; using StabilityMatrix.Core.Extensions; using StabilityMatrix.Core.Helper; using StabilityMatrix.Core.Models; using StabilityMatrix.Core.Models.FileInterfaces; using StabilityMatrix.Core.Processes; namespace StabilityMatrix.Core.Python; /// /// Python runner using a subprocess, mainly for venv support. /// [SuppressMessage("ReSharper", "MemberCanBePrivate.Global")] public class PyVenvRunner : IDisposable, IAsyncDisposable { private static readonly Logger Logger = LogManager.GetCurrentClassLogger(); public const string TorchPipInstallArgsCuda = "torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/cu118"; public const string TorchPipInstallArgsCpu = "torch torchvision torchaudio"; public const string TorchPipInstallArgsDirectML = "torch-directml"; /// /// Relative path to the site-packages folder from the venv root. /// This is platform specific. /// public static string RelativeSitePackagesPath => Compat.Switch( (PlatformKind.Windows, "Lib/site-packages"), (PlatformKind.Unix, "lib/python3.10/site-packages")); /// /// The process running the python executable. /// public AnsiProcess? Process { get; private set; } /// /// The path to the venv root directory. /// public DirectoryPath RootPath { get; } /// /// Name of the python binary folder. /// 'Scripts' on Windows, 'bin' on Unix. /// public static string RelativeBinPath => Compat.Switch( (PlatformKind.Windows, "Scripts"), (PlatformKind.Unix, "bin")); /// /// The relative path to the python executable. /// public static string RelativePythonPath => Compat.Switch( (PlatformKind.Windows, Path.Combine("Scripts", "python.exe")), (PlatformKind.Unix, Path.Combine("bin", "python3"))); /// /// The full path to the python executable. /// public FilePath PythonPath => RootPath.JoinFile(RelativePythonPath); /// /// The relative path to the pip executable. /// public static string RelativePipPath => Compat.Switch( (PlatformKind.Windows, Path.Combine("Scripts", "pip.exe")), (PlatformKind.Unix, Path.Combine("bin", "pip3"))); /// /// The full path to the pip executable. /// public FilePath PipPath => RootPath.JoinFile(RelativePipPath); /// /// List of substrings to suppress from the output. /// When a line contains any of these substrings, it will not be forwarded to callbacks. /// A corresponding Info log will be written instead. /// public List SuppressOutput { get; } = new() { "fatal: not a git repository" }; public PyVenvRunner(DirectoryPath path) { RootPath = path; } /// True if the venv has a Scripts\python.exe file public bool Exists() => PythonPath.Exists; /// /// Creates a venv at the configured path. /// public async Task Setup(bool existsOk = false) { if (!existsOk && Exists()) { throw new InvalidOperationException("Venv already exists"); } // Create RootPath if it doesn't exist RootPath.Create(); // Create venv (copy mode if windows) var args = new string[] { "-m", "virtualenv", Compat.IsWindows ? "--always-copy" : "", RootPath }; var venvProc = ProcessRunner.StartAnsiProcess(PyRunner.PythonExePath, args); await venvProc.WaitForExitAsync(); // Check return code var returnCode = venvProc.ExitCode; if (returnCode != 0) { var output = await venvProc.StandardOutput.ReadToEndAsync(); output += await venvProc.StandardError.ReadToEndAsync(); throw new InvalidOperationException($"Venv creation failed with code {returnCode}: {output}"); } } /// /// Set current python path to pyvenv.cfg /// This should be called before using the venv, in case user moves the venv directory. /// private void SetPyvenvCfg(string pythonDirectory) { // Skip if we are not created yet if (!Exists()) return; // Path to pyvenv.cfg var cfgPath = Path.Combine(RootPath, "pyvenv.cfg"); if (!File.Exists(cfgPath)) { throw new FileNotFoundException("pyvenv.cfg not found", cfgPath); } Logger.Info("Updating pyvenv.cfg with embedded Python directory {PyDir}", pythonDirectory); // Insert a top section var topSection = "[top]" + Environment.NewLine; var cfg = new ConfigParser(topSection + File.ReadAllText(cfgPath)); // Need to set all path keys - home, base-prefix, base-exec-prefix, base-executable cfg.SetValue("top", "home", pythonDirectory); cfg.SetValue("top", "base-prefix", pythonDirectory); cfg.SetValue("top", "base-exec-prefix", pythonDirectory); cfg.SetValue("top", "base-executable", Path.Combine(pythonDirectory, Compat.IsWindows ? "python.exe" : RelativePythonPath)); // Convert to string for writing, strip the top section var cfgString = cfg.ToString()!.Replace(topSection, ""); File.WriteAllText(cfgPath, cfgString); } /// /// Run a pip install command. Waits for the process to exit. /// workingDirectory defaults to RootPath. /// public async Task PipInstall(string args, string? workingDirectory = null, Action? outputDataReceived = null) { if (!File.Exists(PipPath)) { throw new FileNotFoundException("pip not found", PipPath); } // Record output for errors var output = new StringBuilder(); var outputAction = outputDataReceived == null ? null : new Action(s => { Logger.Debug($"Pip output: {s.Text}"); // Record to output output.Append(s.Text); // Forward to callback outputDataReceived(s); }); SetPyvenvCfg(PyRunner.PythonDir); RunDetached($"-m pip install {args}", outputAction, workingDirectory: workingDirectory ?? RootPath); await Process.WaitForExitAsync(); // Check return code if (Process.ExitCode != 0) { throw new ProcessException( $"pip install failed with code {Process.ExitCode}: {output.ToString().ToRepr()}"); } } [MemberNotNull(nameof(Process))] public void RunDetached( string arguments, Action? outputDataReceived, Action? onExit = null, bool unbuffered = true, string workingDirectory = "", Dictionary? environmentVariables = null) { if (!Exists()) { throw new InvalidOperationException("Venv python process does not exist"); } SetPyvenvCfg(PyRunner.PythonDir); Logger.Debug($"Launching RunDetached at {PythonPath} with args {arguments}"); var filteredOutput = outputDataReceived == null ? null : new Action(s => { if (SuppressOutput.Any(s.Text.Contains)) { Logger.Info("Filtered output: {S}", s); return; } outputDataReceived.Invoke(s); }); environmentVariables ??= new Dictionary(); // Disable pip caching - uses significant memory for large packages like torch environmentVariables["PIP_NO_CACHE_DIR"] = "true"; // On windows, add portable git if (Compat.IsWindows) { var portableGit = GlobalConfig.LibraryDir.JoinDir("PortableGit", "bin"); environmentVariables["PATH"] = Compat.GetEnvPathWithExtensions(portableGit); } if (unbuffered) { environmentVariables["PYTHONUNBUFFERED"] = "1"; // If arguments starts with -, it's a flag, insert `u` after it for unbuffered mode if (arguments.StartsWith('-')) { arguments = arguments.Insert(1, "u"); } // Otherwise insert -u at the beginning else { arguments = "-u " + arguments; } } Process = ProcessRunner.StartAnsiProcess(PythonPath, arguments, workingDirectory: workingDirectory, outputDataReceived: filteredOutput, environmentVariables: environmentVariables); if (onExit != null) { Process.EnableRaisingEvents = true; Process.Exited += (sender, _) => { onExit((sender as AnsiProcess)?.ExitCode ?? -1); }; } } public void Dispose() { Process?.CancelStreamReaders(); Process?.Kill(); Process = null; GC.SuppressFinalize(this); } public async ValueTask DisposeAsync() { if (Process is not null) { Process.Kill(); await Process.WaitForExitAsync().ConfigureAwait(false); } Process = null; GC.SuppressFinalize(this); } }