Skip to content

Commit 4f929dd

Browse files
nohwndnohwnd
authored
[rel/17.6] Filter out known platform sources (#4517)
1 parent 3c283e0 commit 4f929dd

File tree

4 files changed

+169
-3
lines changed

4 files changed

+169
-3
lines changed

‎src/Microsoft.TestPlatform.CrossPlatEngine/TestEngine.cs

+1-1
Original file line numberDiff line numberDiff line change
@@ -663,7 +663,7 @@ private static void WarnAboutNotFoundRuntimeProvidersOrThrowWhenNoneAreFound(Lis
663663
throw new ArgumentException(null, nameof(testRuntimeProviders));
664664

665665
// Throw when we did not find any runtime provider for any of the provided sources.
666-
var shouldThrow = testRuntimeProviders.All(runtimeProvider => runtimeProvider == null);
666+
var shouldThrow = testRuntimeProviders.All(runtimeProvider => runtimeProvider.Type == null);
667667

668668
var missingRuntimeProviders = testRuntimeProviders.Where(p => p.Type == null);
669669
if (missingRuntimeProviders.Any())

‎src/vstest.console/CommandLine/CommandLineOptions.cs

+3-1
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
using System.Linq;
99

1010
using Microsoft.VisualStudio.TestPlatform.CommandLine.Processors;
11+
using Microsoft.VisualStudio.TestPlatform.CommandLine.TestPlatformHelpers;
1112
using Microsoft.VisualStudio.TestPlatform.ObjectModel;
1213
using Microsoft.VisualStudio.TestPlatform.Utilities.Helpers;
1314
using Microsoft.VisualStudio.TestPlatform.Utilities.Helpers.Interfaces;
@@ -292,7 +293,8 @@ public void AddSource(string source)
292293
string.Format(CultureInfo.CurrentCulture, CommandLineResources.InvalidArgument, source), ex);
293294
}
294295
// Add the matching files to source list
295-
_sources = _sources.Union(matchingFiles).ToList();
296+
var filteredFiles = KnownPlatformSourceFilter.FilterKnownPlatformSources(matchingFiles);
297+
_sources = _sources.Union(filteredFiles).ToList();
296298
}
297299

298300
/// <summary>

‎src/vstest.console/TestPlatformHelpers/TestRequestManager.cs

+66-1
Original file line numberDiff line numberDiff line change
@@ -167,7 +167,7 @@ public void DiscoverTests(
167167
EqtTrace.Info("TestRequestManager.DiscoverTests: Discovery tests started.");
168168

169169
// TODO: Normalize rest of the data on the request as well
170-
discoveryPayload.Sources = discoveryPayload.Sources?.Distinct().ToList() ?? new List<string>();
170+
discoveryPayload.Sources = KnownPlatformSourceFilter.FilterKnownPlatformSources(discoveryPayload.Sources?.Distinct().ToList());
171171
discoveryPayload.RunSettings ??= "<RunSettings></RunSettings>";
172172

173173
var runsettings = discoveryPayload.RunSettings;
@@ -275,6 +275,11 @@ public void RunTests(
275275
{
276276
EqtTrace.Info("TestRequestManager.RunTests: run tests started.");
277277
testRunRequestPayload.RunSettings ??= "<RunSettings></RunSettings>";
278+
if (testRunRequestPayload.Sources != null)
279+
{
280+
testRunRequestPayload.Sources = KnownPlatformSourceFilter.FilterKnownPlatformSources(testRunRequestPayload.Sources);
281+
}
282+
278283
var runsettings = testRunRequestPayload.RunSettings;
279284

280285
if (testRunRequestPayload.TestPlatformOptions != null)
@@ -1465,3 +1470,63 @@ private static List<string> GetSources(TestRunRequestPayload testRunRequestPaylo
14651470
return sources;
14661471
}
14671472
}
1473+
1474+
internal static class KnownPlatformSourceFilter
1475+
{
1476+
1477+
// Running tests on AzureDevops, many projects use the default filter
1478+
// which includes all *test*.dll, this includes many of the TestPlatform dlls,
1479+
// which we cannot run, and don't want to attempt to run.
1480+
// The default filter also filters out !*TestAdapter*.dll but it is easy to forget
1481+
// so we skip the most used adapters here as well.
1482+
private static readonly HashSet<string> KnownPlatformSources = new(new string[]
1483+
{
1484+
"Microsoft.TestPlatform.CommunicationUtilities.dll",
1485+
"Microsoft.TestPlatform.CoreUtilities.dll",
1486+
"Microsoft.TestPlatform.CrossPlatEngine.dll",
1487+
"Microsoft.TestPlatform.PlatformAbstractions.dll",
1488+
"Microsoft.TestPlatform.Utilities.dll",
1489+
"Microsoft.VisualStudio.TestPlatform.Common.dll",
1490+
"Microsoft.VisualStudio.TestPlatform.ObjectModel.dll",
1491+
"testhost.dll",
1492+
"Microsoft.TestPlatform.AdapterUtilities.dll",
1493+
1494+
// NUnit
1495+
"NUnit3.TestAdapter.dll",
1496+
1497+
// XUnit
1498+
"xunit.runner.visualstudio.testadapter.dll",
1499+
"xunit.runner.visualstudio.dotnetcore.testadapter.dll",
1500+
1501+
// MSTest
1502+
"Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.dll",
1503+
"Microsoft.VisualStudio.TestPlatform.MSTestAdapter.PlatformServices.dll",
1504+
"Microsoft.VisualStudio.TestPlatform.MSTestAdapter.PlatformServices.Interface.dll",
1505+
"Microsoft.VisualStudio.TestPlatform.TestFramework.dll",
1506+
"Microsoft.VisualStudio.TestPlatform.TestFramework.Extensions.dll",
1507+
}, StringComparer.OrdinalIgnoreCase);
1508+
1509+
1510+
internal static List<string> FilterKnownPlatformSources(List<string>? sources)
1511+
{
1512+
if (sources == null)
1513+
{
1514+
return new List<string>();
1515+
}
1516+
1517+
var filteredSources = new List<string>();
1518+
foreach (string source in sources)
1519+
{
1520+
if (KnownPlatformSources.Contains(Path.GetFileName(source)))
1521+
{
1522+
EqtTrace.Info($"TestRequestManager.FilterKnownPlatformSources: Known platform dll was provided in sources, removing it '{source}'");
1523+
}
1524+
else
1525+
{
1526+
filteredSources.Add(source);
1527+
}
1528+
}
1529+
1530+
return filteredSources;
1531+
}
1532+
}

‎test/Microsoft.TestPlatform.AcceptanceTests/ExecutionTests.cs

+99
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,9 @@
99
using Microsoft.VisualStudio.TestPlatform.CoreUtilities.Extensions;
1010

1111
using TestPlatform.TestUtilities;
12+
using System.Linq;
13+
using Microsoft.VisualStudio.TestPlatform.Common;
14+
using FluentAssertions;
1215

1316
namespace Microsoft.TestPlatform.AcceptanceTests;
1417

@@ -396,4 +399,100 @@ public void ExitCodeShouldNotDependOnFailTreatNoTestsAsErrorFalseValueWhenThereA
396399
// Returning 1 because of failing test in test assembly (SimpleTestProject2.dll)
397400
ExitCodeEquals(1);
398401
}
402+
403+
[TestMethod]
404+
[NetFullTargetFrameworkDataSource(inIsolation: true, inProcess: true)]
405+
[NetCoreTargetFrameworkDataSource]
406+
public void RunXunitTestsWhenProvidingAllDllsInBin(RunnerInfo runnerInfo)
407+
{
408+
// This is the default filter of AzDo VSTest task:
409+
// testAssemblyVer2: |
410+
// **\*test *.dll
411+
// ! * *\*TestAdapter.dll
412+
// ! * *\obj\**
413+
// Because of this in typical run we get a lot of dlls that we are sure don't have tests, like Microsoft.VisualStudio.TestPlatform.MSTestAdapter.PlatformServices.dll
414+
// or testhost.dll. Those dlls are built for netcoreapp3.1 tfm, so theoretically they should be tests, but attempting to run them fails to find runtimeconfig.json
415+
// or deps.json, and fails the run.
416+
SetTestEnvironment(_testEnvironment, runnerInfo);
417+
418+
var testAssemblyPath = _testEnvironment.GetTestAsset("XUTestProject.dll");
419+
var allDllsMatchingTestPattern = Directory.GetFiles(Path.GetDirectoryName(testAssemblyPath)!, "*test*.dll");
420+
421+
string assemblyPaths = string.Join(" ", allDllsMatchingTestPattern.Concat(new[] { testAssemblyPath }).Select(s => s.AddDoubleQuote()));
422+
InvokeVsTestForExecution(assemblyPaths, testAdapterPath: string.Empty, FrameworkArgValue, string.Empty);
423+
var fails = this.StdErrWithWhiteSpace.Split('\n').Where(s => !s.IsNullOrWhiteSpace()).Select(s => s.Trim()).ToList();
424+
fails.Should().HaveCount(2, "because there is 1 failed test, and one message that tests failed.");
425+
fails.Last().Should().Be("Test Run Failed.");
426+
}
427+
428+
[TestMethod]
429+
[NetFullTargetFrameworkDataSource(inIsolation: true, inProcess: true)]
430+
[NetCoreTargetFrameworkDataSource()]
431+
public void RunMstestTestsWhenProvidingAllDllsInBin(RunnerInfo runnerInfo)
432+
{
433+
// This is the default filter of AzDo VSTest task:
434+
// testAssemblyVer2: |
435+
// **\*test *.dll
436+
// ! * *\*TestAdapter.dll
437+
// ! * *\obj\**
438+
// Because of this in typical run we get a lot of dlls that we are sure don't have tests, like Microsoft.VisualStudio.TestPlatform.MSTestAdapter.PlatformServices.dll
439+
// or testhost.dll. Those dlls are built for netcoreapp3.1 tfm, so theoretically they should be tests, but attempting to run them fails to find runtimeconfig.json
440+
// or deps.json, and fails the run.
441+
SetTestEnvironment(_testEnvironment, runnerInfo);
442+
443+
var testAssemblyPath = _testEnvironment.GetTestAsset("SimpleTestProject.dll");
444+
var allDllsMatchingTestPattern = Directory.GetFiles(Path.GetDirectoryName(testAssemblyPath)!, "*test*.dll");
445+
446+
string assemblyPaths = string.Join(" ", allDllsMatchingTestPattern.Concat(new[] { testAssemblyPath }).Select(s => s.AddDoubleQuote()));
447+
InvokeVsTestForExecution(assemblyPaths, testAdapterPath: string.Empty, FrameworkArgValue, string.Empty);
448+
449+
StdErrHasTestRunFailedMessageButNoOtherError();
450+
}
451+
452+
[TestMethod]
453+
[NetFullTargetFrameworkDataSource(inIsolation: true, inProcess: true)]
454+
[NetCoreTargetFrameworkDataSource()]
455+
public void RunNunitTestsWhenProvidingAllDllsInBin(RunnerInfo runnerInfo)
456+
{
457+
// This is the default filter of AzDo VSTest task:
458+
// testAssemblyVer2: |
459+
// **\*test *.dll
460+
// ! * *\*TestAdapter.dll
461+
// ! * *\obj\**
462+
// Because of this in typical run we get a lot of dlls that we are sure don't have tests, like Microsoft.VisualStudio.TestPlatform.MSTestAdapter.PlatformServices.dll
463+
// or testhost.dll. Those dlls are built for netcoreapp3.1 tfm, so theoretically they should be tests, but attempting to run them fails to find runtimeconfig.json
464+
// or deps.json, and fails the run.
465+
SetTestEnvironment(_testEnvironment, runnerInfo);
466+
467+
var testAssemblyPath = _testEnvironment.GetTestAsset("NUTestProject.dll");
468+
var allDllsMatchingTestPattern = Directory.GetFiles(Path.GetDirectoryName(testAssemblyPath)!, "*test*.dll");
469+
470+
string assemblyPaths = string.Join(" ", allDllsMatchingTestPattern.Concat(new[] { testAssemblyPath }).Select(s => s.AddDoubleQuote()));
471+
InvokeVsTestForExecution(assemblyPaths, testAdapterPath: string.Empty, FrameworkArgValue, string.Empty);
472+
473+
StdErrHasTestRunFailedMessageButNoOtherError();
474+
}
475+
476+
[TestMethod]
477+
[NetCoreTargetFrameworkDataSource(useDesktopRunner: false)]
478+
public void RunTestsWhenProvidingJustPlatformDllsFailsTheRun(RunnerInfo runnerInfo)
479+
{
480+
// This is the default filter of AzDo VSTest task:
481+
// testAssemblyVer2: |
482+
// **\*test *.dll
483+
// ! * *\*TestAdapter.dll
484+
// ! * *\obj\**
485+
// Because of this in typical run we get a lot of dlls that we are sure don't have tests, like Microsoft.VisualStudio.TestPlatform.MSTestAdapter.PlatformServices.dll
486+
// or testhost.dll. Those dlls are built for netcoreapp3.1 tfm, so theoretically they should be tests, but attempting to run them fails to find runtimeconfig.json
487+
// or deps.json, and fails the run.
488+
SetTestEnvironment(_testEnvironment, runnerInfo);
489+
490+
var xunitAssemblyPath = _testEnvironment.GetTestAsset("SimpleTestProject.dll");
491+
var allDllsMatchingTestPattern = Directory.GetFiles(Path.GetDirectoryName(xunitAssemblyPath)!, "*test*.dll").Where(f => !f.EndsWith("SimpleTestProject.dll"));
492+
493+
string assemblyPaths = string.Join(" ", allDllsMatchingTestPattern.Select(s => s.AddDoubleQuote()));
494+
InvokeVsTestForExecution(assemblyPaths, testAdapterPath: string.Empty, FrameworkArgValue, string.Empty);
495+
496+
StdErr.Should().Be("No test source files were specified. ", "because all platform files we provided were filtered out");
497+
}
399498
}

0 commit comments

Comments
 (0)