diff --git a/source/Calamari.Common/FeatureToggles/OctopusFeatureToggle.cs b/source/Calamari.Common/FeatureToggles/OctopusFeatureToggle.cs index ef680155d..d34c5ccf4 100644 --- a/source/Calamari.Common/FeatureToggles/OctopusFeatureToggle.cs +++ b/source/Calamari.Common/FeatureToggles/OctopusFeatureToggle.cs @@ -7,9 +7,11 @@ public static class OctopusFeatureToggles public static class KnownSlugs { public const string AnsiColorsInTaskLogFeatureToggle = "ansi-colors"; + public const string ArgoCDHelmReplacePathFromContainerReferenceFeatureToggle = "argo-cd-helm-replace-path-from-container-reference"; }; - public static readonly OctopusFeatureToggle AnsiColorsInTaskLogFeatureToggle = new OctopusFeatureToggle(KnownSlugs.AnsiColorsInTaskLogFeatureToggle); + public static readonly OctopusFeatureToggle AnsiColorsInTaskLogFeatureToggle = new(KnownSlugs.AnsiColorsInTaskLogFeatureToggle); + public static readonly OctopusFeatureToggle ArgoCDHelmReplacePathFromContainerReferenceFeatureToggle = new(KnownSlugs.ArgoCDHelmReplacePathFromContainerReferenceFeatureToggle); public class OctopusFeatureToggle { diff --git a/source/Calamari.Common/Plumbing/Variables/PackageVariables.cs b/source/Calamari.Common/Plumbing/Variables/PackageVariables.cs index c22096321..24dd7b018 100644 --- a/source/Calamari.Common/Plumbing/Variables/PackageVariables.cs +++ b/source/Calamari.Common/Plumbing/Variables/PackageVariables.cs @@ -27,7 +27,7 @@ public static class PackageVariables public static string IndexedPackagePurpose(string packageReferenceName) => $"Octopus.Action.Package[{packageReferenceName}].Purpose"; - public static string HelmValueYamlPath(string packageReferenceName) => $"Octopus.Action.Package[{packageReferenceName}].HelmReplacementPath"; + public static string HelmReplacementPath(string packageReferenceName) => $"Octopus.Action.Package[{packageReferenceName}].HelmReplacementPath"; public static string IndexedOriginalPath(string packageReferenceName) { diff --git a/source/Calamari.Tests/ArgoCD/Commands/Conventions/UpdateArgoCDAppImagesInstallConventionHelmTests.cs b/source/Calamari.Tests/ArgoCD/Commands/Conventions/UpdateArgoCDAppImagesInstallConventionHelmTests.cs index e0bf85ada..aafcf2f42 100644 --- a/source/Calamari.Tests/ArgoCD/Commands/Conventions/UpdateArgoCDAppImagesInstallConventionHelmTests.cs +++ b/source/Calamari.Tests/ArgoCD/Commands/Conventions/UpdateArgoCDAppImagesInstallConventionHelmTests.cs @@ -11,6 +11,7 @@ using Calamari.ArgoCD.Git.GitVendorApiAdapters; using Calamari.ArgoCD.Models; using Calamari.Common.Commands; +using Calamari.Common.FeatureToggles; using Calamari.Common.Plumbing.Deployment; using Calamari.Common.Plumbing.FileSystem; using Calamari.Common.Plumbing.Variables; @@ -1348,6 +1349,7 @@ public void DirectorySource_ImageMatches_ReportsDeploymentWithNonEmptyCommitSha( [PackageVariables.IndexedPackagePurpose("nginx")] = "DockerImageReference", [PackageVariables.IndexedImage("alpine")] = "alpine:2.2", [PackageVariables.IndexedPackagePurpose("alpine")] = "DockerImageReference", + [KnownVariables.EnabledFeatureToggles] = OctopusFeatureToggles.KnownSlugs.ArgoCDHelmReplacePathFromContainerReferenceFeatureToggle, }; //Act @@ -1382,6 +1384,171 @@ public void DirectorySource_ImageMatches_ReportsDeploymentWithNonEmptyCommitSha( ]); } + [Test] + public void CanUpdateRefSourceUsingStepBasedVariables() + { + // Arrange + var updater = CreateConvention(); + var variables = new CalamariVariables + { + [PackageVariables.IndexedImage("nginx")] = "index.docker.io/nginx:1.27.1", + [PackageVariables.IndexedPackagePurpose("nginx")] = "DockerImageReference", + [PackageVariables.HelmReplacementPath("nginx")] = "image.nginx", //NOTE: no .Values to start, and no leading . + [ProjectVariables.Slug] = ProjectSlug, + [DeploymentEnvironment.Slug] = EnvironmentSlug, + [KnownVariables.EnabledFeatureToggles] = OctopusFeatureToggles.KnownSlugs.ArgoCDHelmReplacePathFromContainerReferenceFeatureToggle, + }; + var runningDeployment = new RunningDeployment(null, variables); + runningDeployment.CurrentDirectoryProvider = DeploymentWorkingDirectory.StagingDirectory; + runningDeployment.StagingDirectory = tempDirectory; + + + var existingYamlFile = "otherRepoPath/values.yaml"; + var filesInRepo = new (string, string)[] + { + ( + existingYamlFile, + @" +image: + nginx: index.docker.io/nginx:1.0 +containerPort: 8080 +service: + type: LoadBalancer +" + ) + }; + originRepo.AddFilesToBranch(argoCDBranchName, filesInRepo); + + var argoCDAppWithHelmSource = new ArgoCDApplicationBuilder() + .WithName("App1") + .WithAnnotations(new Dictionary() + { + [ArgoCDConstants.Annotations.OctopusProjectAnnotationKey(new ApplicationSourceName("ref-source"))] = ProjectSlug, + [ArgoCDConstants.Annotations.OctopusEnvironmentAnnotationKey(new ApplicationSourceName("ref-source"))] = EnvironmentSlug, + }) + .WithSource(new ApplicationSource + { + OriginalRepoUrl = "https://github.com/org/repo", + Path = "", + TargetRevision = ArgoCDBranchFriendlyName, + Helm = new HelmConfig + { + ValueFiles = new List() + { + "$values/otherRepoPath/values.yaml" + } + }, + Name = "helm-source", + }, + SourceTypeConstants.Helm) + .WithSource(new ApplicationSource + { + Name = "ref-source", + Ref = "values", + TargetRevision = ArgoCDBranchFriendlyName, + OriginalRepoUrl = OriginPath, + }, + SourceTypeConstants.Directory) + .Build(); + + argoCdApplicationManifestParser.ParseManifest(Arg.Any()) + .Returns(argoCDAppWithHelmSource); + // Act + updater.Install(runningDeployment); + + //Assert + const string updatedYamlContent = + @" +image: + nginx: index.docker.io/nginx:1.27.1 +containerPort: 8080 +service: + type: LoadBalancer +"; + + var clonedRepoPath = RepositoryHelpers.CloneOrigin(tempDirectory, OriginPath, argoCDBranchName); + AssertFileContents(clonedRepoPath, existingYamlFile, updatedYamlContent); + } + + + [Test] + public void CanUpdateHelmSourceUsingStepBasedVariables() + { + // Arrange + var updater = CreateConvention(); + var variables = new CalamariVariables + { + [PackageVariables.IndexedImage("nginx")] = "docker.io/nginx:1.27.1", // NOTE the lack of "index" + [PackageVariables.IndexedPackagePurpose("nginx")] = "DockerImageReference", + [PackageVariables.HelmReplacementPath("nginx")] = "image.nginx", //NOTE: no .Values to start, and no leading . + [ProjectVariables.Slug] = ProjectSlug, + [DeploymentEnvironment.Slug] = EnvironmentSlug, + [KnownVariables.EnabledFeatureToggles] = OctopusFeatureToggles.KnownSlugs.ArgoCDHelmReplacePathFromContainerReferenceFeatureToggle, + }; + var runningDeployment = new RunningDeployment(null, variables); + runningDeployment.CurrentDirectoryProvider = DeploymentWorkingDirectory.StagingDirectory; + runningDeployment.StagingDirectory = tempDirectory; + + + var existingYamlFile = "subFolder/values.yaml"; + var filesInRepo = new (string, string)[] + { + ( + existingYamlFile, + @" +image: + nginx: index.docker.io/nginx:1.0 +containerPort: 8080 +service: + type: LoadBalancer +" + ) + }; + originRepo.AddFilesToBranch(argoCDBranchName, filesInRepo); + + var argoCDAppWithHelmSource = new ArgoCDApplicationBuilder() + .WithName("App1") + .WithAnnotations(new Dictionary() + { + [ArgoCDConstants.Annotations.OctopusProjectAnnotationKey(new ApplicationSourceName("helm-source"))] = ProjectSlug, + [ArgoCDConstants.Annotations.OctopusEnvironmentAnnotationKey(new ApplicationSourceName("helm-source"))] = EnvironmentSlug, + }) + .WithSource(new ApplicationSource + { + TargetRevision = ArgoCDBranchFriendlyName, + OriginalRepoUrl = OriginPath, + Path = "", + Helm = new HelmConfig + { + ValueFiles = new List() + { + "subFolder/values.yaml" + } + }, + Name = "helm-source", + }, + SourceTypeConstants.Helm) + .Build(); + + argoCdApplicationManifestParser.ParseManifest(Arg.Any()) + .Returns(argoCDAppWithHelmSource); + // Act + updater.Install(runningDeployment); + + //Assert + const string updatedYamlContent = + @" +image: + nginx: index.docker.io/nginx:1.27.1 +containerPort: 8080 +service: + type: LoadBalancer +"; + + var clonedRepoPath = RepositoryHelpers.CloneOrigin(tempDirectory, OriginPath, argoCDBranchName); + AssertFileContents(clonedRepoPath, existingYamlFile, updatedYamlContent); + } + void AssertFileContents(string clonedRepoPath, string relativeFilePath, string expectedContent) { var absolutePath = Path.Combine(clonedRepoPath, relativeFilePath); @@ -1411,4 +1578,4 @@ void AssertOutputVariables(bool updated = true, string matchingApplicationTotalS } } } -} \ No newline at end of file +} diff --git a/source/Calamari/ArgoCD/Conventions/DeploymentConfigFactory.cs b/source/Calamari/ArgoCD/Conventions/DeploymentConfigFactory.cs index a52b077b0..475c7360d 100644 --- a/source/Calamari/ArgoCD/Conventions/DeploymentConfigFactory.cs +++ b/source/Calamari/ArgoCD/Conventions/DeploymentConfigFactory.cs @@ -36,8 +36,8 @@ public UpdateArgoCDAppDeploymentConfig CreateUpdateImageConfig(RunningDeployment var commitParameters = CommitParameters(deployment); var packageHelmReference = deployment.Variables.GetContainerPackages().Select(p => new ContainerImageReferenceAndHelmReference(ContainerImageReference.FromReferenceString(p.PackageName), p.HelmReference)).ToList(); - - return new UpdateArgoCDAppDeploymentConfig(commitParameters, packageHelmReference); + var useHelmReferenceFromContainer = OctopusFeatureToggles.ArgoCDHelmReplacePathFromContainerReferenceFeatureToggle.IsEnabled(deployment.Variables); + return new UpdateArgoCDAppDeploymentConfig(commitParameters, packageHelmReference, useHelmReferenceFromContainer); } bool RequiresPullRequest(RunningDeployment deployment) diff --git a/source/Calamari/ArgoCD/Conventions/UpdateArgoCDAppDeploymentConfig.cs b/source/Calamari/ArgoCD/Conventions/UpdateArgoCDAppDeploymentConfig.cs index 3525cdb5c..82503f14d 100644 --- a/source/Calamari/ArgoCD/Conventions/UpdateArgoCDAppDeploymentConfig.cs +++ b/source/Calamari/ArgoCD/Conventions/UpdateArgoCDAppDeploymentConfig.cs @@ -1,5 +1,5 @@ using System.Collections.Generic; -using Calamari.ArgoCD.Models; +using System.Linq; namespace Calamari.ArgoCD.Conventions { @@ -7,11 +7,18 @@ public class UpdateArgoCDAppDeploymentConfig { public GitCommitParameters CommitParameters { get; } public List ImageReferences { get; } + public bool UseHelmReferenceFromContainer { get; } - public UpdateArgoCDAppDeploymentConfig(GitCommitParameters commitParameters, List imageReferences) + public UpdateArgoCDAppDeploymentConfig(GitCommitParameters commitParameters, List imageReferences, bool useHelmReferenceFromContainer) { CommitParameters = commitParameters; ImageReferences = imageReferences; + UseHelmReferenceFromContainer = useHelmReferenceFromContainer; + } + + public bool HasStepBasedHelmValueReferences() + { + return ImageReferences.Any(ir => ir.HelmReference is not null) && UseHelmReferenceFromContainer; } } } diff --git a/source/Calamari/ArgoCD/Conventions/UpdateArgoCDAppImagesInstallConvention.cs b/source/Calamari/ArgoCD/Conventions/UpdateArgoCDAppImagesInstallConvention.cs index 3873f294c..3e67dde1e 100644 --- a/source/Calamari/ArgoCD/Conventions/UpdateArgoCDAppImagesInstallConvention.cs +++ b/source/Calamari/ArgoCD/Conventions/UpdateArgoCDAppImagesInstallConvention.cs @@ -66,12 +66,12 @@ public void Install(RunningDeployment deployment) { log.Verbose("Executing Update Argo CD Application Images"); var deploymentConfig = deploymentConfigFactory.CreateUpdateImageConfig(deployment); - + var repositoryFactory = new RepositoryFactory(log, - fileSystem, - deployment.CurrentDirectory, - gitVendorAgnosticApiAdapterFactory, - clock); + fileSystem, + deployment.CurrentDirectory, + gitVendorAgnosticApiAdapterFactory, + clock); var argoProperties = customPropertiesLoader.Load(); @@ -85,11 +85,11 @@ public void Install(RunningDeployment deployment) { var gateway = argoProperties.Gateways.Single(g => g.Id == application.GatewayId); return ProcessApplication(application, - gateway, - deploymentScope, - gitCredentials, - repositoryFactory, - deploymentConfig); + gateway, + deploymentScope, + gitCredentials, + repositoryFactory, + deploymentConfig); }) .ToList(); @@ -103,11 +103,11 @@ public void Install(RunningDeployment deployment) var gatewayIds = argoProperties.Applications.Select(a => a.GatewayId).ToHashSet(); outputVariablesWriter.WriteImageUpdateOutput(gatewayIds, - gitReposUpdated, - totalApplicationsWithSourceCounts, - updatedApplicationsWithSources, - newImagesWritten.Count - ); + gitReposUpdated, + totalApplicationsWithSourceCounts, + updatedApplicationsWithSources, + newImagesWritten.Count + ); } ProcessApplicationResult ProcessApplication( @@ -125,18 +125,27 @@ ProcessApplicationResult ProcessApplication( ValidateApplication(applicationFromYaml); + var imagesWithNoHelmReference = deploymentConfig.ImageReferences.Where(c => c.HelmReference is null).ToList(); + if (imagesWithNoHelmReference.Any() && applicationFromYaml.GetSourcesWithMetadata().Any(src => src.SourceType == SourceType.Helm)) + { + foreach (var image in imagesWithNoHelmReference) + { + log.Verbose($"{image.ContainerReference.ToString()} will not be updated in helm sources, as no helm yaml path has been specified for it in the step configuration."); + } + } + var updatedSourcesResults = applicationFromYaml.GetSourcesWithMetadata() .Select(applicationSource => new { Updated = ProcessSource(applicationSource, - applicationFromYaml, - containsMultipleSources, - deploymentScope, - gitCredentials, - repositoryFactory, - deploymentConfig, - application.DefaultRegistry, - gateway), + applicationFromYaml, + containsMultipleSources, + deploymentScope, + gitCredentials, + repositoryFactory, + deploymentConfig, + application.DefaultRegistry, + gateway), applicationSource, }) .Where(r => r.Updated.ImagesUpdated.Any()) @@ -155,13 +164,13 @@ ProcessApplicationResult ProcessApplication( log.InfoFormat(message, linkifiedAppName); return new ProcessApplicationResult( - application.GatewayId, - applicationName.ToApplicationName(), - applicationFromYaml.Spec.Sources.Count, - applicationFromYaml.Spec.Sources.Count(s => deploymentScope.Matches(ScopingAnnotationReader.GetScopeForApplicationSource(s.Name.ToApplicationSourceName(), applicationFromYaml.Metadata.Annotations, containsMultipleSources))), - updatedSourcesResults.Select(r => new UpdatedSourceDetail(r.Updated.CommitSha, r.applicationSource.Index, [], r.Updated.PatchedFiles)).ToList(), - updatedSourcesResults.SelectMany(r => r.Updated.ImagesUpdated).ToHashSet(), - updatedSourcesResults.Select(r => r.applicationSource.Source.OriginalRepoUrl).ToHashSet()); + application.GatewayId, + applicationName.ToApplicationName(), + applicationFromYaml.Spec.Sources.Count, + applicationFromYaml.Spec.Sources.Count(s => deploymentScope.Matches(ScopingAnnotationReader.GetScopeForApplicationSource(s.Name.ToApplicationSourceName(), applicationFromYaml.Metadata.Annotations, containsMultipleSources))), + updatedSourcesResults.Select(r => new UpdatedSourceDetail(r.Updated.CommitSha, r.applicationSource.Index, [], r.Updated.PatchedFiles)).ToList(), + updatedSourcesResults.SelectMany(r => r.Updated.ImagesUpdated).ToHashSet(), + updatedSourcesResults.Select(r => r.applicationSource.Source.OriginalRepoUrl).ToHashSet()); } void ValidateApplication(Application applicationFromYaml) @@ -200,39 +209,39 @@ SourceUpdateResult ProcessSource( { return applicationSource.Ref != null ? ProcessRef(applicationFromYaml, - gitCredentials, - repositoryFactory, - deploymentConfig, - sourceWithMetadata, - defaultRegistry, - gateway) + gitCredentials, + repositoryFactory, + deploymentConfig, + sourceWithMetadata, + defaultRegistry, + gateway) : ProcessDirectory(applicationFromYaml, - gitCredentials, - repositoryFactory, - deploymentConfig, - sourceWithMetadata, - defaultRegistry, - gateway); + gitCredentials, + repositoryFactory, + deploymentConfig, + sourceWithMetadata, + defaultRegistry, + gateway); } case SourceType.Helm: { return ProcessHelm(applicationFromYaml, - sourceWithMetadata, - gitCredentials, - repositoryFactory, - deploymentConfig, - defaultRegistry, - gateway); + sourceWithMetadata, + gitCredentials, + repositoryFactory, + deploymentConfig, + defaultRegistry, + gateway); } case SourceType.Kustomize: { return ProcessKustomize(applicationFromYaml, - gitCredentials, - repositoryFactory, - deploymentConfig, - sourceWithMetadata, - defaultRegistry, - gateway); + gitCredentials, + repositoryFactory, + deploymentConfig, + sourceWithMetadata, + defaultRegistry, + gateway); } case SourceType.Plugin: { @@ -270,17 +279,17 @@ SourceUpdateResult ProcessKustomize( if (updatedImages.Count > 0) { var pushResult = PushToRemote(repository, - GitReference.CreateFromString(applicationSource.TargetRevision), - deploymentConfig.CommitParameters, - updatedFiles, - updatedImages); + GitReference.CreateFromString(applicationSource.TargetRevision), + deploymentConfig.CommitParameters, + updatedFiles, + updatedImages); if (pushResult is not null) { outputVariablesWriter.WritePushResultOutput(gateway.Name, - applicationFromYaml.Metadata.Name, - sourceWithMetadata.Index, - pushResult); + applicationFromYaml.Metadata.Name, + sourceWithMetadata.Index, + pushResult); return new SourceUpdateResult(updatedImages, pushResult.CommitSha, patchedFiles); } } @@ -306,19 +315,34 @@ SourceUpdateResult ProcessRef( log.WarnFormat("The source '{0}' contains a Ref, only referenced files will be updated. Please create another source with the same URL if you wish to update files under the path.", sourceWithMetadata.SourceIdentity); } + using var repository = CreateRepository(gitCredentials, applicationSource, repositoryFactory); + if (deploymentConfig.HasStepBasedHelmValueReferences()) + { + if (applicationFromYaml.Metadata.Annotations.ContainsKey(ArgoCDConstants.Annotations.OctopusImageReplacementPathsKey(new ApplicationSourceName(sourceWithMetadata.Source.Name)))) + { + log.Warn($"Application {applicationFromYaml.Metadata.Name} specifies helm-value annotations which have been superseded by container-values specified in the step's configuration"); + } + + return ProcessRefSourceUsingStepVariables(applicationFromYaml, + sourceWithMetadata, + repository, + deploymentConfig, + defaultRegistry, + gateway); + } + var helmTargetsForRefSource = new HelmValuesFileUpdateTargetParser(applicationFromYaml, defaultRegistry) .GetHelmTargetsForRefSource(sourceWithMetadata); LogHelmSourceConfigurationProblems(helmTargetsForRefSource.Problems); - using var repository = CreateRepository(gitCredentials, applicationSource, repositoryFactory); return ProcessHelmUpdateTargets( - applicationFromYaml, - repository, - deploymentConfig, - sourceWithMetadata, - helmTargetsForRefSource.Targets, - gateway); + applicationFromYaml, + repository, + deploymentConfig, + sourceWithMetadata, + helmTargetsForRefSource.Targets, + gateway); } /// Images that were updated @@ -346,17 +370,17 @@ SourceUpdateResult ProcessDirectory( if (updatedImages.Count > 0) { var pushResult = PushToRemote(repository, - GitReference.CreateFromString(applicationSource.TargetRevision), - deploymentConfig.CommitParameters, - updatedFiles, - updatedImages); + GitReference.CreateFromString(applicationSource.TargetRevision), + deploymentConfig.CommitParameters, + updatedFiles, + updatedImages); if (pushResult is not null) { outputVariablesWriter.WritePushResultOutput(gateway.Name, - applicationFromYaml.Metadata.Name, - sourceWithMetadata.Index, - pushResult); + applicationFromYaml.Metadata.Name, + sourceWithMetadata.Index, + pushResult); return new SourceUpdateResult(updatedImages, pushResult.CommitSha, patchedFiles); } @@ -385,6 +409,42 @@ SourceUpdateResult ProcessHelm( return new SourceUpdateResult(new HashSet(), string.Empty, []); } + if (deploymentConfig.HasStepBasedHelmValueReferences()) + { + var appName = sourceWithMetadata.Source.Name.IsNullOrEmpty() ? null : new ApplicationSourceName(sourceWithMetadata.Source.Name); + if (applicationFromYaml.Metadata.Annotations.ContainsKey(ArgoCDConstants.Annotations.OctopusImageReplacementPathsKey(appName))) + { + log.Warn($"Application '{applicationFromYaml.Metadata.Name}' specifies helm-value annotations which have been superseded by values specified in the step's configuration"); + } + + return ProcessHelmSourceUsingStepVariables(applicationFromYaml, + gitCredentials, + repositoryFactory, + deploymentConfig, + sourceWithMetadata, + defaultRegistry, + gateway); + } + + return ProcessHelmSourceUsingAnnotations(applicationFromYaml, + sourceWithMetadata, + gitCredentials, + repositoryFactory, + deploymentConfig, + defaultRegistry, + gateway, + applicationSource); + } + + SourceUpdateResult ProcessHelmSourceUsingAnnotations(Application applicationFromYaml, + ApplicationSourceWithMetadata sourceWithMetadata, + Dictionary gitCredentials, + RepositoryFactory repositoryFactory, + UpdateArgoCDAppDeploymentConfig deploymentConfig, + string defaultRegistry, + ArgoCDGatewayDto gateway, + ApplicationSource applicationSource) + { var explicitHelmSources = new HelmValuesFileUpdateTargetParser(applicationFromYaml, defaultRegistry) .GetExplicitValuesFilesToUpdate(sourceWithMetadata); @@ -398,9 +458,9 @@ SourceUpdateResult ProcessHelm( if (implicitValuesFile != null && explicitHelmSources.Targets.None(t => t.FileName == implicitValuesFile)) { var (target, problem) = AddImplicitValuesFile(applicationFromYaml, - sourceWithMetadata, - implicitValuesFile, - defaultRegistry); + sourceWithMetadata, + implicitValuesFile, + defaultRegistry); if (target != null) valuesFilesToUpdate.Add(target); @@ -411,11 +471,11 @@ SourceUpdateResult ProcessHelm( LogHelmSourceConfigurationProblems(valueFileProblems); return ProcessHelmUpdateTargets(applicationFromYaml, - repository, - deploymentConfig, - sourceWithMetadata, - valuesFilesToUpdate, - gateway); + repository, + deploymentConfig, + sourceWithMetadata, + valuesFilesToUpdate, + gateway); } /// Images that were updated @@ -428,32 +488,117 @@ SourceUpdateResult ProcessHelmUpdateTargets( ArgoCDGatewayDto gateway) { var results = targets.Select(t => UpdateHelmImageValues(repository.WorkingDirectory, - t, - deploymentConfig.ImageReferences - )) + t, + deploymentConfig.ImageReferences + )) .ToList(); var updatedImages = results.SelectMany(r => r.ImagesUpdated).ToHashSet(); if (updatedImages.Count > 0) { var patchedFiles = results.Select(r => new FilePathContent( - // Replace \ with / so that Calamari running on windows doesn't cause issues when we send back to server - r.RelativeFilepath.Replace('\\', '/'), - r.JsonPatch is not null ? JsonSerializer.Serialize(r.JsonPatch) : null)) + // Replace \ with / so that Calamari running on windows doesn't cause issues when we send back to server + r.RelativeFilepath.Replace('\\', '/'), + r.JsonPatch is not null ? JsonSerializer.Serialize(r.JsonPatch) : null)) .ToList(); var pushResult = PushToRemote(repository, - GitReference.CreateFromString(sourceWithMetadata.Source.TargetRevision), - deploymentConfig.CommitParameters, - results.Where(r => r.ImagesUpdated.Any()).Select(r => r.RelativeFilepath).ToHashSet(), - updatedImages); + GitReference.CreateFromString(sourceWithMetadata.Source.TargetRevision), + deploymentConfig.CommitParameters, + results.Where(r => r.ImagesUpdated.Any()).Select(r => r.RelativeFilepath).ToHashSet(), + updatedImages); + + if (pushResult is not null) + { + outputVariablesWriter.WritePushResultOutput(gateway.Name, + applicationFromYaml.Metadata.Name, + sourceWithMetadata.Index, + pushResult); + return new SourceUpdateResult(updatedImages, pushResult.CommitSha, patchedFiles); + } + } + + return new SourceUpdateResult(new HashSet(), string.Empty, []); + } + + SourceUpdateResult ProcessHelmSourceUsingStepVariables( + Application applicationFromYaml, + Dictionary gitCredentials, + RepositoryFactory repositoryFactory, + UpdateArgoCDAppDeploymentConfig deploymentConfig, + ApplicationSourceWithMetadata sourceWithMetadata, + string defaultRegistry, + ArgoCDGatewayDto gateway) + { + var extractor = new HelmValuesFileExtractor(applicationFromYaml); + var valuesFilesInHelmSource = extractor.GetInlineValuesFilesReferencedByHelmSource(sourceWithMetadata); + + using var repository = CreateRepository(gitCredentials, sourceWithMetadata.Source, repositoryFactory); + var filesToUpdate = valuesFilesInHelmSource.Select(file => Path.Combine(repository.WorkingDirectory, file)).ToList(); + var implicitValuesFile = HelmDiscovery.TryFindValuesFile(fileSystem, Path.Combine(repository.WorkingDirectory, sourceWithMetadata.Source.Path!)); + if (implicitValuesFile != null) + { + implicitValuesFile = Path.Combine(repository.WorkingDirectory, sourceWithMetadata.Source.Path!, implicitValuesFile); + filesToUpdate.Add(implicitValuesFile); + } + filesToUpdate = filesToUpdate.Select(file => Path.Combine(repository.WorkingDirectory, file)).ToList(); + var result = ProcessHelmValuesFiles(filesToUpdate.ToHashSet(), + defaultRegistry, + repository, + deploymentConfig, + gateway, + sourceWithMetadata, + applicationFromYaml); + return result; + } + + + SourceUpdateResult ProcessRefSourceUsingStepVariables(Application applicationFromYaml, + ApplicationSourceWithMetadata sourceWithMetadata, + RepositoryWrapper repository, + UpdateArgoCDAppDeploymentConfig deploymentConfig, + string defaultRegistry, + ArgoCDGatewayDto gateway) + { + var extractor = new HelmValuesFileExtractor(applicationFromYaml); + var valuesFiles = extractor.GetValueFilesReferencedInRefSource(sourceWithMetadata) + .Select(file => Path.Combine(repository.WorkingDirectory, file)); + return ProcessHelmValuesFiles(valuesFiles.ToHashSet(), + defaultRegistry, + repository, + deploymentConfig, + gateway, + sourceWithMetadata, + applicationFromYaml); + } + + SourceUpdateResult ProcessHelmValuesFiles(HashSet filesToUpdate, + string defaultRegistry, + RepositoryWrapper repository, + UpdateArgoCDAppDeploymentConfig deploymentConfig, + ArgoCDGatewayDto gateway, + ApplicationSourceWithMetadata sourceWithMetadata, + Application applicationFromYaml) + { + Func imageReplacerFactory = yaml => new HelmValuesImageReplaceStepVariables(yaml, defaultRegistry, log); + log.Verbose($"Found {filesToUpdate.Count} yaml files to process"); + + var (updatedFiles, updatedImages, patchedFiles) = Update(repository.WorkingDirectory, deploymentConfig.ImageReferences, filesToUpdate.ToHashSet(), imageReplacerFactory); + if (updatedImages.Count > 0) + { + Log.Info("Trying to push up changes"); + var pushResult = PushToRemote(repository, + GitReference.CreateFromString(sourceWithMetadata.Source.TargetRevision), + deploymentConfig.CommitParameters, + updatedFiles, + updatedImages); if (pushResult is not null) { outputVariablesWriter.WritePushResultOutput(gateway.Name, - applicationFromYaml.Metadata.Name, - sourceWithMetadata.Index, - pushResult); + applicationFromYaml.Metadata.Name, + sourceWithMetadata.Index, + pushResult); return new SourceUpdateResult(updatedImages, pushResult.CommitSha, patchedFiles); } } @@ -477,13 +622,13 @@ void LogProblem(HelmSourceConfigurationProblem helmSourceConfigurationProblem) if (helmSourceIsMissingImagePathAnnotation.RefSourceIdentity == null) { log.WarnFormat("The Helm source '{0}' is missing an annotation for the image replace path. It will not be updated.", - helmSourceIsMissingImagePathAnnotation.SourceIdentity); + helmSourceIsMissingImagePathAnnotation.SourceIdentity); } else { log.WarnFormat("The Helm source '{0}' is missing an annotation for the image replace path. The source '{1}' will not be updated.", - helmSourceIsMissingImagePathAnnotation.SourceIdentity, - helmSourceIsMissingImagePathAnnotation.RefSourceIdentity); + helmSourceIsMissingImagePathAnnotation.SourceIdentity, + helmSourceIsMissingImagePathAnnotation.RefSourceIdentity); } log.WarnFormat("Annotation creation documentation can be found {0}.", log.FormatShortLink("argo-cd-helm-image-annotations", "here")); @@ -515,18 +660,18 @@ RepositoryWrapper CreateRepository(Dictionary gitCrede string defaultRegistry) { var imageReplacePaths = ScopingAnnotationReader.GetImageReplacePathsForApplicationSource( - applicationSource.Source.Name.ToApplicationSourceName(), - applicationFromYaml.Metadata.Annotations, - applicationFromYaml.Spec.Sources.Count > 1); + applicationSource.Source.Name.ToApplicationSourceName(), + applicationFromYaml.Metadata.Annotations, + applicationFromYaml.Spec.Sources.Count > 1); if (!imageReplacePaths.Any()) { return (null, new HelmSourceIsMissingImagePathAnnotation(applicationSource.SourceIdentity)); } return (new HelmValuesFileImageUpdateTarget(defaultRegistry, - applicationSource.Source.Path, - valuesFilename, - imageReplacePaths), null); + applicationSource.Source.Path, + valuesFilename, + imageReplacePaths), null); } (HashSet, HashSet, List) UpdateKubernetesYaml( @@ -643,10 +788,10 @@ HelmRefUpdatedResult UpdateHelmImageValues( log.Verbose("Pushing to remote"); return repository.PushChanges(commitParameters.RequiresPr, - commitParameters.Summary, - commitDescription, - branchName, - CancellationToken.None) + commitParameters.Summary, + commitDescription, + branchName, + CancellationToken.None) .GetAwaiter() .GetResult(); } @@ -678,4 +823,4 @@ string Serialize(JsonPatchDocument patchDocument) record SourceUpdateResult(HashSet ImagesUpdated, string CommitSha, List PatchedFiles); } -} \ No newline at end of file +} diff --git a/source/Calamari/ArgoCD/Helm/HelmValuesFileExtractor.cs b/source/Calamari/ArgoCD/Helm/HelmValuesFileExtractor.cs new file mode 100644 index 000000000..f9a6944be --- /dev/null +++ b/source/Calamari/ArgoCD/Helm/HelmValuesFileExtractor.cs @@ -0,0 +1,40 @@ +#nullable enable +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using Calamari.ArgoCD.Conventions; +using Calamari.ArgoCD.Domain; +using Calamari.ArgoCD.Models; + +namespace Calamari.ArgoCD.Helm +{ + public class HelmValuesFileExtractor + { + readonly List helmSources; + + public HelmValuesFileExtractor(Application toUpdate) + { + helmSources = toUpdate.GetSourcesWithMetadata().Where(s => s.SourceType == SourceType.Helm).ToList(); + } + + public IReadOnlyCollection GetInlineValuesFilesReferencedByHelmSource(ApplicationSourceWithMetadata helmSource) + { + return helmSource.Source.Helm?.ValueFiles.Where(file => !file.StartsWith('$')) + .Select(vf => Path.Combine(helmSource.Source.Path!, vf)) + .ToList() + ?? []; + } + + public IReadOnlyCollection GetValueFilesReferencedInRefSource(ApplicationSourceWithMetadata refSource) + { + var refPrefix = $"${refSource.Source.Ref!}/"; + return helmSources + .Where(hs => hs.Source.Helm != null) + .SelectMany(hs => hs.Source.Helm!.ValueFiles.Where(f => f.StartsWith(refPrefix))) + .Distinct() + .Select(vf => vf.Substring(refPrefix.Length)) + .ToList(); + } + } +} diff --git a/source/Calamari/ArgoCD/HelmValuesImageReplaceStepVariables.cs b/source/Calamari/ArgoCD/HelmValuesImageReplaceStepVariables.cs new file mode 100644 index 000000000..24bd71c1b --- /dev/null +++ b/source/Calamari/ArgoCD/HelmValuesImageReplaceStepVariables.cs @@ -0,0 +1,73 @@ +using System.Collections.Generic; +using System.Linq; +using Calamari.ArgoCD.Conventions; +using Calamari.ArgoCD.Helm; +using Calamari.ArgoCD.Models; +using Calamari.Common.Plumbing.Logging; + +namespace Calamari.ArgoCD; + +public class HelmValuesImageReplaceStepVariables : IContainerImageReplacer +{ + readonly string yamlContent; + readonly string defaultRegistry; + readonly ILog log; + + public HelmValuesImageReplaceStepVariables(string yamlContent, string defaultRegistry, ILog log) + { + this.yamlContent = yamlContent; + this.defaultRegistry = defaultRegistry; + this.log = log; + } + + public ImageReplacementResult UpdateImages(IReadOnlyCollection imagesToUpdate) + { + var imagesUpdated = new HashSet(); + var updatedYaml = yamlContent; + var originalYamlParser = new HelmYamlParser(yamlContent); // Parse and track the original yaml so that content can be read from it. + var flattenedYamlPathDictionary = HelmValuesEditor.GenerateVariableDictionary(originalYamlParser); + + foreach (var newImageTag in imagesToUpdate.Where(c => c.HelmReference is not null)) + { + var helmReference = newImageTag.HelmReference!; + var valueToUpdate = flattenedYamlPathDictionary.GetRaw(helmReference); + if (valueToUpdate == null) + { + log.Verbose($"{helmReference} for image {newImageTag.ContainerReference.ToString()} was not found in your values file."); + continue; + } + + if (IsUnstructuredText(valueToUpdate)) + { + HelmValuesEditor.UpdateNodeValue(updatedYaml, helmReference, newImageTag.ContainerReference.Tag); + } + else + { + var cir = ContainerImageReference.FromReferenceString(valueToUpdate, defaultRegistry); + var comparison = newImageTag.ContainerReference.CompareWith(cir); + if (comparison.MatchesImage()) + { + if (!comparison.TagMatch) + { + var newValue = cir.WithTag(newImageTag.ContainerReference.Tag); + updatedYaml = HelmValuesEditor.UpdateNodeValue(updatedYaml, helmReference, newValue); + imagesUpdated.Add(newValue); + } + } + else + { + log.Warn($"Attempted to update value entry '{helmReference}', however it contains a mismatched image name and registry."); + } + } + } + return new ImageReplacementResult(updatedYaml, imagesUpdated); + } + + bool IsUnstructuredText(string content) + { + var lastColonIndex = content.LastIndexOf(':'); + var lastSlashIndex = content.LastIndexOf('/'); + + return lastColonIndex == -1 && lastSlashIndex == -1; + } +} \ No newline at end of file diff --git a/source/Calamari/ArgoCD/VariablesExtensionMethods.cs b/source/Calamari/ArgoCD/VariablesExtensionMethods.cs index 2320cb362..f11872a09 100644 --- a/source/Calamari/ArgoCD/VariablesExtensionMethods.cs +++ b/source/Calamari/ArgoCD/VariablesExtensionMethods.cs @@ -16,7 +16,7 @@ public static IList GetContainerPackages(this IVariable var packageReferences = (from packageIndex in packageIndexes let image = variables.Get(PackageVariables.IndexedImage(packageIndex), string.Empty) let purpose = variables.Get(PackageVariables.IndexedPackagePurpose(packageIndex), string.Empty) - let helmValueYamlPath = variables.Get(PackageVariables.HelmValueYamlPath(packageIndex), null) + let helmValueYamlPath = variables.Get(PackageVariables.HelmReplacementPath(packageIndex), null) where purpose.Equals("DockerImageReference", StringComparison.Ordinal) select new PackageAndHelmReference(image, helmValueYamlPath)) .ToList();