Skip to content

Commit b7f9806

Browse files
bradwilsonbradwilson
committed
Fixes #332: Handling null/throwing TestCollectionOrder
1 parent 8efae51 commit b7f9806

5 files changed

Lines changed: 121 additions & 14 deletions

File tree

‎src/xunit.execution/Sdk/Frameworks/Runners/TestAssemblyRunner.cs‎

Lines changed: 14 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -149,13 +149,24 @@ protected virtual IMessageBus CreateMessageBus()
149149
/// Orders the test collections using the <see cref="TestCollectionOrderer"/>.
150150
/// </summary>
151151
/// <returns>Test collections (and the associated test cases) in run order</returns>
152-
protected List<Tuple<ITestCollection, List<TTestCase>>> OrderTestCases()
152+
protected List<Tuple<ITestCollection, List<TTestCase>>> OrderTestCollections()
153153
{
154154
var testCasesByCollection =
155155
TestCases.GroupBy(tc => tc.TestMethod.TestClass.TestCollection, TestCollectionComparer.Instance)
156156
.ToDictionary(collectionGroup => collectionGroup.Key, collectionGroup => collectionGroup.ToList());
157157

158-
var orderedTestCollections = TestCollectionOrderer.OrderTestCollections(testCasesByCollection.Keys);
158+
IEnumerable<ITestCollection> orderedTestCollections;
159+
160+
try
161+
{
162+
orderedTestCollections = TestCollectionOrderer.OrderTestCollections(testCasesByCollection.Keys);
163+
}
164+
catch (Exception ex)
165+
{
166+
var innerEx = ex.Unwrap();
167+
DiagnosticMessageSink.OnMessage(new DiagnosticMessage("Test collection orderer '{0}' threw '{1}' during ordering: {2}", TestCollectionOrderer.GetType().FullName, innerEx.GetType().FullName, innerEx.StackTrace));
168+
orderedTestCollections = testCasesByCollection.Keys.ToList();
169+
}
159170

160171
return orderedTestCollections.Select(collection => Tuple.Create(collection, testCasesByCollection[collection]))
161172
.ToList();
@@ -221,7 +232,7 @@ protected virtual async Task<RunSummary> RunTestCollectionsAsync(IMessageBus mes
221232
{
222233
var summary = new RunSummary();
223234

224-
foreach (var collection in OrderTestCases())
235+
foreach (var collection in OrderTestCollections())
225236
{
226237
summary.Aggregate(await RunTestCollectionAsync(messageBus, collection.Item1, collection.Item2, cancellationTokenSource));
227238
if (cancellationTokenSource.IsCancellationRequested)

‎src/xunit.execution/Sdk/Frameworks/Runners/XunitTestAssemblyRunner.cs‎

Lines changed: 20 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -123,7 +123,25 @@ protected void Initialize()
123123

124124
var testCollectionOrdererAttribute = TestAssembly.Assembly.GetCustomAttributes(typeof(TestCollectionOrdererAttribute)).SingleOrDefault();
125125
if (testCollectionOrdererAttribute != null)
126-
TestCollectionOrderer = ExtensibilityPointFactory.GetTestCollectionOrderer(DiagnosticMessageSink, testCollectionOrdererAttribute);
126+
{
127+
try
128+
{
129+
var testCollectionOrderer = ExtensibilityPointFactory.GetTestCollectionOrderer(DiagnosticMessageSink, testCollectionOrdererAttribute);
130+
if (testCollectionOrderer != null)
131+
TestCollectionOrderer = testCollectionOrderer;
132+
else
133+
{
134+
var args = testCollectionOrdererAttribute.GetConstructorArguments().Cast<string>().ToList();
135+
DiagnosticMessageSink.OnMessage(new DiagnosticMessage("Could not find type '{0}' in {1} for assembly-level test collection orderer", args[0], args[1]));
136+
}
137+
}
138+
catch (Exception ex)
139+
{
140+
var innerEx = ex.Unwrap();
141+
var args = testCollectionOrdererAttribute.GetConstructorArguments().Cast<string>().ToList();
142+
DiagnosticMessageSink.OnMessage(new DiagnosticMessage("Assembly-level test collection orderer '{0}' threw '{1}' during construction: {2}", args[0], innerEx.GetType().FullName, innerEx.StackTrace));
143+
}
144+
}
127145

128146
initialized = true;
129147
}
@@ -154,7 +172,7 @@ protected override async Task<RunSummary> RunTestCollectionsAsync(IMessageBus me
154172

155173
var scheduler = TaskScheduler.FromCurrentSynchronizationContext();
156174

157-
var tasks = OrderTestCases().Select(
175+
var tasks = OrderTestCollections().Select(
158176
collection => Task.Factory.StartNew(() => RunTestCollectionAsync(messageBus, collection.Item1, collection.Item2, cancellationTokenSource),
159177
cancellationTokenSource.Token,
160178
TaskCreationOptions.DenyChildAttach | TaskCreationOptions.HideScheduler,

‎test/test.utility/Mocks.cs‎

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -260,17 +260,20 @@ public static TestCollection TestCollection(Assembly assembly = null, ITypeInfo
260260
return new TestCollection(Mocks.TestAssembly(assembly), definition, displayName);
261261
}
262262

263-
public static IReflectionAttributeInfo TestCollectionOrdererAttribute<TOrderer>()
263+
public static IReflectionAttributeInfo TestCollectionOrdererAttribute(string typeName, string assemblyName)
264264
{
265265
var result = Substitute.For<IReflectionAttributeInfo, InterfaceProxy<IReflectionAttributeInfo>>();
266-
var ordererType = typeof(TOrderer);
267-
var typeName = ordererType.FullName;
268-
var assemblyName = ordererType.Assembly.FullName;
269266
result.Attribute.Returns(new TestCollectionOrdererAttribute(typeName, assemblyName));
270267
result.GetConstructorArguments().Returns(new object[] { typeName, assemblyName });
271268
return result;
272269
}
273270

271+
public static IReflectionAttributeInfo TestCollectionOrdererAttribute<TOrderer>()
272+
{
273+
var ordererType = typeof(TOrderer);
274+
return TestCollectionOrdererAttribute(ordererType.FullName, ordererType.Assembly.FullName);
275+
}
276+
274277
public static ITestFailed TestFailed(Type type, string methodName, string displayName = null, string output = null, decimal executionTime = 0M, Exception ex = null)
275278
{
276279
var testCase = Mocks.TestCase(type, methodName);

‎test/test.xunit.execution/Sdk/Frameworks/Runners/TestAssemblyRunnerTests.cs‎

Lines changed: 39 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -275,6 +275,38 @@ public IEnumerable<ITestCollection> OrderTestCollections(IEnumerable<ITestCollec
275275
return TestCollections.OrderByDescending(c => c.DisplayName);
276276
}
277277
}
278+
279+
[Fact]
280+
public static async void TestCaseOrdererWhichThrowsLogsMessageAndDoesNotReorderTests()
281+
{
282+
var collection1 = Mocks.TestCollection(displayName: "AAA");
283+
var testCase1 = Mocks.TestCase(collection1);
284+
var collection2 = Mocks.TestCollection(displayName: "ZZZZ");
285+
var testCase2 = Mocks.TestCase(collection2);
286+
var collection3 = Mocks.TestCollection(displayName: "MM");
287+
var testCase3 = Mocks.TestCase(collection3);
288+
var testCases = new[] { testCase1, testCase2, testCase3 };
289+
var runner = TestableTestAssemblyRunner.Create(testCases: testCases);
290+
runner.TestCollectionOrderer = new ThrowingOrderer();
291+
292+
await runner.RunAsync();
293+
294+
Assert.Collection(runner.CollectionsRun,
295+
collection => Assert.Same(collection1, collection.Item1),
296+
collection => Assert.Same(collection2, collection.Item1),
297+
collection => Assert.Same(collection3, collection.Item1)
298+
);
299+
var diagnosticMessage = Assert.Single(runner.DiagnosticMessages.Cast<IDiagnosticMessage>());
300+
Assert.StartsWith("Test collection orderer 'TestAssemblyRunnerTests+TestCollectionOrderer+ThrowingOrderer' threw 'System.DivideByZeroException' during ordering:", diagnosticMessage.Message);
301+
}
302+
303+
class ThrowingOrderer : ITestCollectionOrderer
304+
{
305+
public IEnumerable<ITestCollection> OrderTestCollections(IEnumerable<ITestCollection> testCollections)
306+
{
307+
throw new DivideByZeroException();
308+
}
309+
}
278310
}
279311

280312
class TestableTestAssemblyRunner : TestAssemblyRunner<ITestCase>
@@ -287,17 +319,20 @@ class TestableTestAssemblyRunner : TestAssemblyRunner<ITestCase>
287319
public bool AfterTestAssemblyStarting_Called;
288320
public Action<ExceptionAggregator> BeforeTestAssemblyFinished_Callback = _ => { };
289321
public bool BeforeTestAssemblyFinished_Called;
322+
public List<IMessageSinkMessage> DiagnosticMessages;
290323
public Exception RunTestCollectionAsync_AggregatorResult;
291324

292325
TestableTestAssemblyRunner(ITestAssembly testAssembly,
293326
IEnumerable<ITestCase> testCases,
294-
IMessageSink diagnosticMessageSink,
327+
List<IMessageSinkMessage> diagnosticMessages,
295328
IMessageSink executionMessageSink,
296329
ITestFrameworkExecutionOptions executionOptions,
297330
RunSummary result,
298331
bool cancelInRunTestCollectionAsync)
299-
: base(testAssembly, testCases, diagnosticMessageSink, executionMessageSink, executionOptions)
332+
: base(testAssembly, testCases, SpyMessageSink.Create(messages: diagnosticMessages), executionMessageSink, executionOptions)
300333
{
334+
DiagnosticMessages = diagnosticMessages;
335+
301336
this.result = result;
302337
this.cancelInRunTestCollectionAsync = cancelInRunTestCollectionAsync;
303338
}
@@ -306,13 +341,12 @@ public static TestableTestAssemblyRunner Create(IMessageSink executionMessageSin
306341
RunSummary result = null,
307342
ITestCase[] testCases = null,
308343
ITestFrameworkExecutionOptions executionOptions = null,
309-
bool cancelInRunTestCollectionAsync = false,
310-
IMessageSink diagnosticMessageSink = null)
344+
bool cancelInRunTestCollectionAsync = false)
311345
{
312346
return new TestableTestAssemblyRunner(
313347
Mocks.TestAssembly(Assembly.GetExecutingAssembly()),
314348
testCases ?? new[] { Substitute.For<ITestCase>() }, // Need at least one so it calls RunTestCollectionAsync
315-
diagnosticMessageSink ?? SpyMessageSink.Create(),
349+
new List<IMessageSinkMessage>(),
316350
executionMessageSink ?? SpyMessageSink.Create(),
317351
executionOptions ?? TestFrameworkOptions.ForExecution(),
318352
result ?? new RunSummary(),

‎test/test.xunit.execution/Sdk/Frameworks/Runners/XunitTestAssemblyRunnerTests.cs‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -271,6 +271,47 @@ public IEnumerable<ITestCollection> OrderTestCollections(IEnumerable<ITestCollec
271271
return TestCollections.OrderByDescending(c => c.DisplayName);
272272
}
273273
}
274+
275+
[Fact]
276+
public static void SettingUnknownTestCollectionOrderLogsDiagnosticMessage()
277+
{
278+
var ordererAttribute = Mocks.TestCollectionOrdererAttribute("UnknownType", "UnknownAssembly");
279+
var assembly = Mocks.TestAssembly(new[] { ordererAttribute });
280+
var runner = TestableXunitTestAssemblyRunner.Create(assembly: assembly);
281+
282+
runner.Initialize();
283+
284+
Assert.IsType<DefaultTestCollectionOrderer>(runner.TestCollectionOrderer);
285+
var diagnosticMessage = Assert.Single(runner.DiagnosticMessages.Cast<IDiagnosticMessage>());
286+
Assert.Equal("Could not find type 'UnknownType' in UnknownAssembly for assembly-level test collection orderer", diagnosticMessage.Message);
287+
}
288+
289+
[Fact]
290+
public static void SettingTestCollectionOrdererWithThrowingConstructorLogsDiagnosticMessage()
291+
{
292+
var ordererAttribute = Mocks.TestCollectionOrdererAttribute<MyCtorThrowingTestCollectionOrderer>();
293+
var assembly = Mocks.TestAssembly(new[] { ordererAttribute });
294+
var runner = TestableXunitTestAssemblyRunner.Create(assembly: assembly);
295+
296+
runner.Initialize();
297+
298+
Assert.IsType<DefaultTestCaseOrderer>(runner.TestCaseOrderer);
299+
var diagnosticMessage = Assert.Single(runner.DiagnosticMessages.Cast<IDiagnosticMessage>());
300+
Assert.StartsWith("Assembly-level test collection orderer 'XunitTestAssemblyRunnerTests+TestCollectionOrderer+MyCtorThrowingTestCollectionOrderer' threw 'System.DivideByZeroException' during construction:", diagnosticMessage.Message);
301+
}
302+
303+
class MyCtorThrowingTestCollectionOrderer : ITestCollectionOrderer
304+
{
305+
public MyCtorThrowingTestCollectionOrderer()
306+
{
307+
throw new DivideByZeroException();
308+
}
309+
310+
public IEnumerable<ITestCollection> OrderTestCollections(IEnumerable<ITestCollection> testCollections)
311+
{
312+
return Enumerable.Empty<ITestCollection>();
313+
}
314+
}
274315
}
275316

276317
class ClassUnderTest

0 commit comments

Comments
 (0)