Skip to content

Fix concurrent Pipeline Cache publication correctness #173

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
David Federman (dfederm) merged 1 commit into from
Aug 7, 2026
Merged

Fix concurrent Pipeline Cache publication correctness #173

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions MSBuildCache.sln
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.MSBuildCache.Loca
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.MSBuildCache.Repack.Tests", "src\Repack.Tests\Microsoft.MSBuildCache.Repack.Tests.csproj", "{3BCB6452-B087-4A03-8418-C79F2715DDE7}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.MSBuildCache.AzurePipelines.Tests", "src\AzurePipelines.Tests\Microsoft.MSBuildCache.AzurePipelines.Tests.csproj", "{61A86AEA-F043-4CC4-B60B-A040C5C36194}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|x64 = Debug|x64
Expand Down Expand Up @@ -68,6 +70,10 @@ Global
{3BCB6452-B087-4A03-8418-C79F2715DDE7}.Debug|x64.Build.0 = Debug|x64
{3BCB6452-B087-4A03-8418-C79F2715DDE7}.Release|x64.ActiveCfg = Release|x64
{3BCB6452-B087-4A03-8418-C79F2715DDE7}.Release|x64.Build.0 = Release|x64
{61A86AEA-F043-4CC4-B60B-A040C5C36194}.Debug|x64.ActiveCfg = Debug|x64
{61A86AEA-F043-4CC4-B60B-A040C5C36194}.Debug|x64.Build.0 = Debug|x64
{61A86AEA-F043-4CC4-B60B-A040C5C36194}.Release|x64.ActiveCfg = Release|x64
{61A86AEA-F043-4CC4-B60B-A040C5C36194}.Release|x64.Build.0 = Release|x64
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
Expand All @@ -80,6 +86,7 @@ Global
{97357681-C75E-445D-8547-46F312D01CED} = {EFFB5949-347C-4F28-8964-571D5C6B6209}
{F6586428-E047-42C8-B0AC-048DF6DFAF18} = {EFFB5949-347C-4F28-8964-571D5C6B6209}
{3BCB6452-B087-4A03-8418-C79F2715DDE7} = {EFFB5949-347C-4F28-8964-571D5C6B6209}
{61A86AEA-F043-4CC4-B60B-A040C5C36194} = {EFFB5949-347C-4F28-8964-571D5C6B6209}
EndGlobalSection
GlobalSection(ExtensibilityGlobals) = postSolution
SolutionGuid = {F1CDA78F-A666-431B-BF44-56DA7DF193BA}
Expand Down
14 changes: 14 additions & 0 deletions src/AzurePipelines.Tests/Microsoft.MSBuildCache.AzurePipelines.Tests.csproj
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
<Project Sdk="MSTest.Sdk">
<PropertyGroup>
<!-- Only supports x64 due to the RocksDB dependency -->
<Platform>x64</Platform>
<Platforms>$(Platform)</Platforms>
<TargetFramework>net9.0</TargetFramework>
<RootNamespace>Microsoft.MSBuildCache.AzurePipelines.Tests</RootNamespace>
<!-- MSTest requires public test classes even though the test host is an application. -->
<NoWarn>$(NoWarn);CA1515</NoWarn>
</PropertyGroup>
<ItemGroup>
<ProjectReference Include="..\AzurePipelines\Microsoft.MSBuildCache.AzurePipelines.csproj" />
</ItemGroup>
</Project>
162 changes: 162 additions & 0 deletions src/AzurePipelines.Tests/SelectorPublicationTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,162 @@
// Copyright (c) Microsoft. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.

using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using BuildXL.Cache.ContentStore.Hashing;
using BuildXL.Cache.MemoizationStore.Interfaces.Sessions;
using Microsoft.VisualStudio.TestTools.UnitTesting;

namespace Microsoft.MSBuildCache.AzurePipelines.Tests;

[TestClass]
public class SelectorPublicationTests
{
[TestMethod]
public async Task ConcurrentPublicationsMergeTheConflictWinner()
{
Selector existingSelector = CreateSelector(1);
Selector firstSelector = CreateSelector(2);
Selector secondSelector = CreateSelector(3);
Fingerprint weakFingerprint = new("01");
const string Universe = "universe";
string latestKey = PipelineCachingCacheClient.ComputeSelectorsReadKey(Universe, weakFingerprint);
PublicationState initial = new("initial", new[] { existingSelector });
ConcurrentDictionary<string, PublicationState> entries = new();
ConcurrentQueue<string> conflictQueries = new();
PublicationState? firstWinner = null;
PublicationState? finalWinner = null;
string? finalWriteKey = null;
int initialAttempts = 0;
TaskCompletionSource<bool> bothInitialAttemptsStarted = new(TaskCreationOptions.RunContinuationsAsynchronously);
TaskCompletionSource<bool> firstWinnerReady = new(TaskCreationOptions.RunContinuationsAsynchronously);

Task<PublicationState?> QueryAsync(string key, CancellationToken cancellationToken)
{
cancellationToken.ThrowIfCancellationRequested();
if (key == latestKey)
{
return Task.FromResult<PublicationState?>(initial);
}

entries.TryGetValue(key, out PublicationState? entry);
return Task.FromResult(entry);
}

Task<PublicationState> GetConflictWinnerAsync(string key, CancellationToken cancellationToken)
{
cancellationToken.ThrowIfCancellationRequested();
conflictQueries.Enqueue(key);
Assert.IsTrue(entries.TryGetValue(key, out PublicationState? entry));
return Task.FromResult(entry);
}

async Task<bool> TryPublishAsync(
PublicationState? predecessor,
IReadOnlyCollection<Selector> selectors,
string key,
CancellationToken cancellationToken)
{
Assert.IsNotNull(predecessor);

if (predecessor.Id == initial.Id)
{
if (Interlocked.Increment(ref initialAttempts) == 2)
{
bothInitialAttemptsStarted.SetResult(true);
}

await bothInitialAttemptsStarted.Task;

if (selectors.Contains(firstSelector))
{
firstWinner = new PublicationState("first", selectors);
Assert.IsTrue(entries.TryAdd(key, firstWinner));
firstWinnerReady.SetResult(true);
return true;
}

await firstWinnerReady.Task;
return false;
}

Assert.AreEqual(firstWinner!.Id, predecessor.Id);
finalWinner = new PublicationState("final", selectors);
finalWriteKey = key;
Assert.IsTrue(entries.TryAdd(key, finalWinner));
return true;
}

Task<bool> firstPublication = SelectorPublication.AddAsync(
firstSelector,
latestKey,
QueryAsync,
GetConflictWinnerAsync,
(state, _) => Task.FromResult<IReadOnlyCollection<Selector>>(state!.Selectors),
state => PipelineCachingCacheClient.ComputeSelectorsWriteKey(Universe, weakFingerprint, state?.Id),
TryPublishAsync,
CancellationToken.None);
Task<bool> secondPublication = SelectorPublication.AddAsync(
secondSelector,
latestKey,
QueryAsync,
GetConflictWinnerAsync,
(state, _) => Task.FromResult<IReadOnlyCollection<Selector>>(state!.Selectors),
state => PipelineCachingCacheClient.ComputeSelectorsWriteKey(Universe, weakFingerprint, state?.Id),
TryPublishAsync,
CancellationToken.None);

Assert.IsTrue(await firstPublication);
Assert.IsTrue(await secondPublication);
CollectionAssert.AreEquivalent(
new[] { existingSelector, firstSelector, secondSelector },
finalWinner!.Selectors.ToArray());
string initialWriteKey = PipelineCachingCacheClient.ComputeSelectorsWriteKey(Universe, weakFingerprint, initial.Id);
CollectionAssert.AreEqual(new[] { initialWriteKey }, conflictQueries.ToArray());
Assert.AreEqual(
PipelineCachingCacheClient.ComputeSelectorsWriteKey(Universe, weakFingerprint, firstWinner!.Id),
finalWriteKey);
StringAssert.StartsWith(latestKey, "selector6|", StringComparison.Ordinal);
}

[TestMethod]
public void OutputKeyIncludesSelectorOutput()
{
Selector firstSelector = CreateSelector(1);
Selector secondSelector = CreateSelector(2);
Fingerprint weakFingerprint = new("01");
StrongFingerprint first = new(weakFingerprint, firstSelector);
StrongFingerprint second = new(weakFingerprint, secondSelector);

string firstKey = PipelineCachingCacheClient.ComputeOutputKey("universe", first, forWrite: false, writeId: 0);
string secondKey = PipelineCachingCacheClient.ComputeOutputKey("universe", second, forWrite: false, writeId: 0);

Assert.AreNotEqual(firstKey, secondKey);
StringAssert.StartsWith(firstKey, "outputs6|", StringComparison.Ordinal);
StringAssert.Contains(firstKey, "|01|", StringComparison.Ordinal);
StringAssert.Contains(secondKey, "|02|", StringComparison.Ordinal);
}

private static Selector CreateSelector(byte output)
{
byte[] hashBytes = Enumerable.Repeat((byte)0x5a, 32).ToArray();
return new Selector(new ContentHash(HashType.SHA256, hashBytes), new[] { output });
}

private sealed class PublicationState
{
public PublicationState(string id, IEnumerable<Selector> selectors)
{
Id = id;
Selectors = new HashSet<Selector>(selectors);
}

public string Id { get; }

public HashSet<Selector> Selectors { get; }
}
}
3 changes: 3 additions & 0 deletions src/AzurePipelines/Microsoft.MSBuildCache.AzurePipelines.csproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,9 @@
<ItemGroup>
<ProjectReference Include="..\Common\Microsoft.MSBuildCache.Common.csproj" />
</ItemGroup>
<ItemGroup>
<InternalsVisibleTo Include="Microsoft.MSBuildCache.AzurePipelines.Tests" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="Microsoft.VisualStudio.Services.PipelineCache.WebApi" />
<PackageReference Include="Microsoft.VisualStudio.Services.BlobStore.Client" />
Expand Down
Loading