From 1ce2afacb32f30194f4ba3896c5bdb1ccdf4307a Mon Sep 17 00:00:00 2001 From: Rhys Parry Date: Wed, 25 Feb 2026 10:04:00 +1000 Subject: [PATCH] Add script isolation enforcement --- .../Calamari.Common/CalamariFlavourProgram.cs | 2 + .../CalamariFlavourProgramAsync.cs | 3 + .../Processes/ScriptIsolation/FileLock.cs | 45 +++++++++ .../Processes/ScriptIsolation/ILockHandle.cs | 5 + .../Processes/ScriptIsolation/Isolation.cs | 99 +++++++++++++++++++ .../Processes/ScriptIsolation/LockOptions.cs | 81 +++++++++++++++ .../ScriptIsolation/LockRejectedException.cs | 30 ++++++ .../Processes/ScriptIsolation/LockType.cs | 7 ++ 8 files changed, 272 insertions(+) create mode 100644 source/Calamari.Common/Features/Processes/ScriptIsolation/FileLock.cs create mode 100644 source/Calamari.Common/Features/Processes/ScriptIsolation/ILockHandle.cs create mode 100644 source/Calamari.Common/Features/Processes/ScriptIsolation/Isolation.cs create mode 100644 source/Calamari.Common/Features/Processes/ScriptIsolation/LockOptions.cs create mode 100644 source/Calamari.Common/Features/Processes/ScriptIsolation/LockRejectedException.cs create mode 100644 source/Calamari.Common/Features/Processes/ScriptIsolation/LockType.cs diff --git a/source/Calamari.Common/CalamariFlavourProgram.cs b/source/Calamari.Common/CalamariFlavourProgram.cs index ac49a301d..fa98b6ae8 100644 --- a/source/Calamari.Common/CalamariFlavourProgram.cs +++ b/source/Calamari.Common/CalamariFlavourProgram.cs @@ -12,6 +12,7 @@ using Calamari.Common.Features.FunctionScriptContributions; using Calamari.Common.Features.Packages; using Calamari.Common.Features.Processes; +using Calamari.Common.Features.Processes.ScriptIsolation; using Calamari.Common.Features.Scripting; using Calamari.Common.Features.Scripting.DotnetScript; using Calamari.Common.Features.StructuredVariables; @@ -78,6 +79,7 @@ protected virtual int Run(string[] args) } #endif + using var _ = Isolation.Enforce(); return ResolveAndExecuteCommand(container, options); } catch (Exception ex) diff --git a/source/Calamari.Common/CalamariFlavourProgramAsync.cs b/source/Calamari.Common/CalamariFlavourProgramAsync.cs index 615342b4b..1be8e9e0d 100644 --- a/source/Calamari.Common/CalamariFlavourProgramAsync.cs +++ b/source/Calamari.Common/CalamariFlavourProgramAsync.cs @@ -3,6 +3,7 @@ using System.Diagnostics; using System.Linq; using System.Reflection; +using System.Threading; using System.Threading.Tasks; using Autofac; using Autofac.Core; @@ -14,6 +15,7 @@ using Calamari.Common.Features.FunctionScriptContributions; using Calamari.Common.Features.Packages; using Calamari.Common.Features.Processes; +using Calamari.Common.Features.Processes.ScriptIsolation; using Calamari.Common.Features.Scripting; using Calamari.Common.Features.Scripting.DotnetScript; using Calamari.Common.Features.StructuredVariables; @@ -143,6 +145,7 @@ protected async Task Run(string[] args) } #endif + await using var _ = await Isolation.EnforceAsync(CancellationToken.None); await ResolveAndExecuteCommand(container, options); return 0; } diff --git a/source/Calamari.Common/Features/Processes/ScriptIsolation/FileLock.cs b/source/Calamari.Common/Features/Processes/ScriptIsolation/FileLock.cs new file mode 100644 index 000000000..6269fe40e --- /dev/null +++ b/source/Calamari.Common/Features/Processes/ScriptIsolation/FileLock.cs @@ -0,0 +1,45 @@ +using System; +using System.IO; +using System.Threading.Tasks; + +namespace Calamari.Common.Features.Processes.ScriptIsolation; + +public static class FileLock +{ + public static ILockHandle Acquire(LockOptions lockOptions) + { + var fileShareMode = GetFileShareMode(lockOptions.Type); + try + { + var fileStream = File.Open(lockOptions.LockFile, FileMode.OpenOrCreate, FileAccess.ReadWrite, fileShareMode); + return new LockHandle(fileStream); + } + catch (IOException e) + { + throw new LockRejectedException(e); + } + } + + static FileShare GetFileShareMode(LockType isolationLevel) + { + return isolationLevel switch + { + LockType.Exclusive => FileShare.None, + LockType.Shared => FileShare.ReadWrite, + _ => throw new ArgumentOutOfRangeException(nameof(isolationLevel), isolationLevel, null) + }; + } + + sealed class LockHandle(FileStream fileStream) : ILockHandle + { + public void Dispose() + { + fileStream.Dispose(); + } + + public async ValueTask DisposeAsync() + { + await fileStream.DisposeAsync(); + } + } +} diff --git a/source/Calamari.Common/Features/Processes/ScriptIsolation/ILockHandle.cs b/source/Calamari.Common/Features/Processes/ScriptIsolation/ILockHandle.cs new file mode 100644 index 000000000..41c69d1b6 --- /dev/null +++ b/source/Calamari.Common/Features/Processes/ScriptIsolation/ILockHandle.cs @@ -0,0 +1,5 @@ +using System; + +namespace Calamari.Common.Features.Processes.ScriptIsolation; + +public interface ILockHandle : IAsyncDisposable, IDisposable; diff --git a/source/Calamari.Common/Features/Processes/ScriptIsolation/Isolation.cs b/source/Calamari.Common/Features/Processes/ScriptIsolation/Isolation.cs new file mode 100644 index 000000000..8b33fb1bf --- /dev/null +++ b/source/Calamari.Common/Features/Processes/ScriptIsolation/Isolation.cs @@ -0,0 +1,99 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using Calamari.Common.Plumbing.Logging; +using Polly; + +namespace Calamari.Common.Features.Processes.ScriptIsolation; + +public static class Isolation +{ + // Compare these values with the standard script isolation mutex strategy + static readonly TimeSpan RetryInitialDelay = TimeSpan.FromMilliseconds(10); + static readonly TimeSpan RetryMaxDelay = TimeSpan.FromMilliseconds(500); + + public static ILockHandle Enforce() + { + var lockOptions = LockOptions.FromEnvironmentOrNull(); + if (lockOptions is null) + { + Log.Verbose("No lock required"); + return new NoLock(); + } + + var pipeline = BuildLockAcquisitionPipeline(lockOptions); + LogIsolation(lockOptions); + try + { + return pipeline.Execute(FileLock.Acquire, lockOptions); + } + catch (Exception exception) + { + LockRejectedException.Throw(exception); + throw; // Satisfy the compiler + } + } + + public static async Task EnforceAsync(CancellationToken cancellationToken) + { + var lockOptions = LockOptions.FromEnvironmentOrNull(); + if (lockOptions is null) + { + return new NoLock(); + } + + var pipeline = BuildLockAcquisitionPipeline(lockOptions); + LogIsolation(lockOptions); + try + { + return await pipeline.ExecuteAsync(static (o, _) => ValueTask.FromResult(FileLock.Acquire(o)), lockOptions, cancellationToken); + } + catch (Exception exception) + { + LockRejectedException.Throw(exception); + throw; // Satisfy the compiler + } + } + + static void LogIsolation(LockOptions lockOptions) + { + Log.Verbose($"Acquiring script isolation mutex {lockOptions.Name} with {lockOptions.Type} lock"); + } + + static ResiliencePipeline BuildLockAcquisitionPipeline(LockOptions lockOptions) + { + var builder = new ResiliencePipelineBuilder(); + // Timeout must be between 10ms and 1 day. (Polly) + // If it's 10ms or less, we'll skip timeout and limit retries + // If it's more than 1 day, we'll assume indefinite retries with no timeout + var retryAttempts = lockOptions.Timeout <= TimeSpan.FromMilliseconds(10) + ? 1 + : int.MaxValue; + if (lockOptions.Timeout < TimeSpan.FromDays(1) && lockOptions.Timeout > TimeSpan.FromMilliseconds(10)) + { + builder.AddTimeout(lockOptions.Timeout); + } + + builder.AddRetry( + new() + { + BackoffType = DelayBackoffType.Exponential, + Delay = RetryInitialDelay, + MaxDelay = RetryMaxDelay, + MaxRetryAttempts = retryAttempts, + ShouldHandle = new PredicateBuilder().Handle(), + UseJitter = true + } + ); + return builder.Build(); + } + + class NoLock : ILockHandle + { + public ValueTask DisposeAsync() => ValueTask.CompletedTask; + + public void Dispose() + { + } + } +} diff --git a/source/Calamari.Common/Features/Processes/ScriptIsolation/LockOptions.cs b/source/Calamari.Common/Features/Processes/ScriptIsolation/LockOptions.cs new file mode 100644 index 000000000..9146a0ab0 --- /dev/null +++ b/source/Calamari.Common/Features/Processes/ScriptIsolation/LockOptions.cs @@ -0,0 +1,81 @@ +using System; +using Calamari.Common.Plumbing.Logging; + +namespace Calamari.Common.Features.Processes.ScriptIsolation; + +public sealed record LockOptions( + LockType Type, + string Name, + string LockFile, + TimeSpan Timeout +) +{ + public static LockOptions? FromEnvironmentOrNull() + { + Log.Verbose("Attempting to load LockOptions from environment variables"); + + var envLevel = GetAndLogEnvironmentVariable(EnvironmentVariables.CalamariScriptIsolationLevel); + var envMutexName = GetAndLogEnvironmentVariable(EnvironmentVariables.CalamariScriptIsolationMutexName); + var envMutexTimeout = GetAndLogEnvironmentVariable(EnvironmentVariables.CalamariScriptIsolationMutexTimeout); + var tentacleHome = GetAndLogEnvironmentVariable(EnvironmentVariables.TentacleHome); + + if (string.IsNullOrWhiteSpace(envLevel) || string.IsNullOrWhiteSpace(envMutexName) || string.IsNullOrWhiteSpace(envMutexTimeout) || string.IsNullOrWhiteSpace(tentacleHome)) + { + // This is for initial debugging - This will indicate that Calamari shouldn't perform locking + Log.Verbose("One or more required environment variables are missing or empty. Unable to create LockOptions."); + return null; + } + + var lockType = MapScriptIsolationLevelToLockTypeOrNull(envLevel); + if (lockType == null) + { + Log.Verbose($"Failed to map script isolation level '{envLevel}' to a valid LockType. Expected 'FullIsolation' or 'NoIsolation' (case-insensitive)."); + return null; + } + + Log.Verbose($"Mapped isolation level '{envLevel}' to LockType.{lockType.Value}"); + + if (!TimeSpan.TryParse(envMutexTimeout, out var timeout)) + { + Log.Verbose($"Failed to parse mutex timeout value '{envMutexTimeout}' as TimeSpan. Defaulting to TimeSpan.MaxValue."); + // What should we do if the timeout is invalid? Default to max value? + timeout = TimeSpan.MaxValue; + } + else + { + Log.Verbose($"Parsed mutex timeout: {timeout}"); + } + + var lockFilePath = GetLockFilePath(tentacleHome, envMutexName); + Log.Verbose($"Calculated lock file path: {lockFilePath}"); + + Log.Verbose($"Successfully created LockOptions with Type={lockType.Value}, Name={envMutexName}, LockFile={lockFilePath}, Timeout={timeout}"); + return new LockOptions(lockType.Value, envMutexName, lockFilePath, timeout); + } + + static string? GetAndLogEnvironmentVariable(string environmentVariableName) + { + var result = Environment.GetEnvironmentVariable(environmentVariableName); + Log.Verbose($"Environment variable '{environmentVariableName}': {(string.IsNullOrWhiteSpace(result) ? "" : result)}"); + return result; + } + + static string GetLockFilePath(string tentacleHome, string mutexName) => + System.IO.Path.Combine(tentacleHome, $"ScriptIsolation.{mutexName}.lock"); // Should we sanitize the mutex name or just allow it to be invalid? + + static LockType? MapScriptIsolationLevelToLockTypeOrNull(string isolationLevel) => + isolationLevel.ToLowerInvariant() switch + { + "fullisolation" => LockType.Exclusive, + "noisolation" => LockType.Shared, + _ => null + }; + + static class EnvironmentVariables + { + public const string CalamariScriptIsolationLevel = "CalamariScriptIsolationLevel"; + public const string CalamariScriptIsolationMutexName = "CalamariScriptIsolationMutexName"; + public const string CalamariScriptIsolationMutexTimeout = "CalamariScriptIsolationMutexTimeout"; + public const string TentacleHome = "TentacleHome"; + } +} diff --git a/source/Calamari.Common/Features/Processes/ScriptIsolation/LockRejectedException.cs b/source/Calamari.Common/Features/Processes/ScriptIsolation/LockRejectedException.cs new file mode 100644 index 000000000..7a9798e7a --- /dev/null +++ b/source/Calamari.Common/Features/Processes/ScriptIsolation/LockRejectedException.cs @@ -0,0 +1,30 @@ +using System; +using System.Diagnostics.CodeAnalysis; +using Polly.Timeout; + +namespace Calamari.Common.Features.Processes.ScriptIsolation; + +public sealed class LockRejectedException(string message, Exception? innerException) + : Exception(message, innerException) +{ + public LockRejectedException(Exception innerException) : this("Lock acquisition failed", innerException) + { + } + + [DoesNotReturn] + public static void Throw(Exception innerException) + { + if (innerException is LockRejectedException lockRejectedException) + { + throw lockRejectedException; + } + + if (innerException is TimeoutRejectedException timeoutRejectedException) + { + var message = $"Lock acquisition failed after {timeoutRejectedException.Timeout}"; + throw new LockRejectedException(message, timeoutRejectedException); + } + + throw new LockRejectedException("Lock acquisition failed", innerException); + } +} diff --git a/source/Calamari.Common/Features/Processes/ScriptIsolation/LockType.cs b/source/Calamari.Common/Features/Processes/ScriptIsolation/LockType.cs new file mode 100644 index 000000000..5104b175f --- /dev/null +++ b/source/Calamari.Common/Features/Processes/ScriptIsolation/LockType.cs @@ -0,0 +1,7 @@ +namespace Calamari.Common.Features.Processes.ScriptIsolation; + +public enum LockType +{ + Shared, + Exclusive +}