using System;
using System.Collections.Generic;
using System.Dynamic;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using Xunit.Abstractions;
using Xunit.Runner.Wpf;
using Xunit.Sdk;
namespace Xunit.Runners
{
///
/// A class which makes it simpler for casual runner authors to find and run tests and get results.
/// Adapted from: https://github.com/xunit/xunit/blob/main/src/xunit.v3.runner.utility/Runners/AssemblyRunner.cs
///
public class AssemblyRunner2 : LongLivedMarshalByRefObject, IDisposable, IMessageSinkWithTypes
{
static readonly Dictionary MessageTypeNames;
private TaskCompletionSource tcs { get; set; }
private CancellationToken cancellationToken { get; set; }
volatile bool cancelled;
bool disposed;
readonly TestAssemblyConfiguration configuration;
readonly IFrontController controller;
string assemblyFileName { get; set; }
int testCasesDiscovered;
readonly List testCasesToRun = new List();
static AssemblyRunner2()
{
MessageTypeNames = new Dictionary();
AddMessageTypeName();
AddMessageTypeName();
AddMessageTypeName();
AddMessageTypeName();
AddMessageTypeName();
AddMessageTypeName();
AddMessageTypeName();
AddMessageTypeName();
AddMessageTypeName();
AddMessageTypeName();
AddMessageTypeName();
AddMessageTypeName();
AddMessageTypeName();
AddMessageTypeName();
AddMessageTypeName();
AddMessageTypeName();
AddMessageTypeName();
}
AssemblyRunner2(AppDomainSupport appDomainSupport,
string assemblyFileName,
TaskCompletionSource tcs,
CancellationToken cancellationToken,
string configFileName = null,
bool shadowCopy = true,
string shadowCopyFolder = null)
{
this.tcs = tcs;
this.cancellationToken = cancellationToken;
this.assemblyFileName = assemblyFileName;
controller = new XunitFrontController(appDomainSupport, assemblyFileName, configFileName, shadowCopy, shadowCopyFolder, diagnosticMessageSink: MessageSinkAdapter.Wrap(this));
configuration = ConfigReader.Load(assemblyFileName, configFileName);
}
///
/// Set to get notification of diagnostic messages.
///
public Action OnDiagnosticMessage { get; set; }
///
/// Set to get notification of when test discovery is complete.
///
public Action OnDiscoveryComplete { get; set; }
///
/// Set to get notification of error messages (unhandled exceptions outside of tests).
///
public Action OnErrorMessage { get; set; }
///
/// Set to get notification of when test execution is complete.
///
public Action OnExecutionComplete { get; set; }
///
/// Set to get notification of failed tests.
///
public Action OnTestFailed { get; set; }
///
/// Set to get notification of finished tests (regardless of outcome).
///
public Action OnTestFinished { get; set; }
///
/// Set to get real-time notification of test output (for xUnit.net v2 tests only).
/// Note that output is captured and reported back to all the test completion Info>s
/// in addition to being sent to this Info>.
///
public Action OnTestOutput { get; set; }
///
/// Set to get notification of passing tests.
///
public Action OnTestPassed { get; set; }
///
/// Set to get notification of skipped tests.
///
public Action OnTestSkipped { get; set; }
///
/// Set to get notification of when tests start running.
///
public Action OnTestStarting { get; set; }
///
/// Set to be able to filter the test cases to decide which ones to run. If this is not set,
/// then all test cases will be run.
///
public Func TestCaseFilter { get; set; }
static void AddMessageTypeName() => MessageTypeNames.Add(typeof(T), typeof(T).FullName);
///
/// Call to request that the current run be cancelled. Note that cancellation may not be
/// instantaneous, and even after cancellation has been acknowledged, you can expect to
/// receive all the cleanup-related messages.
///
public void Cancel()
{
cancelled = true;
}
///
public void Dispose()
{
if (disposed)
return;
disposed = true;
controller?.Dispose();
}
ITestFrameworkDiscoveryOptions GetDiscoveryOptions(bool? diagnosticMessages, TestMethodDisplay? methodDisplay, TestMethodDisplayOptions? methodDisplayOptions, bool? preEnumerateTheories, bool? internalDiagnosticMessages)
{
var discoveryOptions = TestFrameworkOptions.ForDiscovery(configuration);
discoveryOptions.SetSynchronousMessageReporting(true);
if (diagnosticMessages.HasValue)
discoveryOptions.SetDiagnosticMessages(diagnosticMessages);
if (internalDiagnosticMessages.HasValue)
discoveryOptions.SetDiagnosticMessages(internalDiagnosticMessages);
if (methodDisplay.HasValue)
discoveryOptions.SetMethodDisplay(methodDisplay);
if (methodDisplayOptions.HasValue)
discoveryOptions.SetMethodDisplayOptions(methodDisplayOptions);
if (preEnumerateTheories.HasValue)
discoveryOptions.SetPreEnumerateTheories(preEnumerateTheories);
return discoveryOptions;
}
ITestFrameworkExecutionOptions GetExecutionOptions(bool? diagnosticMessages, bool? parallel, int? maxParallelThreads, bool? internalDiagnosticMessages)
{
var executionOptions = TestFrameworkOptions.ForExecution(configuration);
executionOptions.SetSynchronousMessageReporting(true);
if (diagnosticMessages.HasValue)
executionOptions.SetDiagnosticMessages(diagnosticMessages);
if (internalDiagnosticMessages.HasValue)
executionOptions.SetDiagnosticMessages(internalDiagnosticMessages);
if (parallel.HasValue)
executionOptions.SetDisableParallelization(!parallel.GetValueOrDefault());
if (maxParallelThreads.HasValue)
executionOptions.SetMaxParallelThreads(maxParallelThreads);
return executionOptions;
}
///
/// Starts running tests from a single type (if provided) or the whole assembly (if not). This call returns
/// immediately, and status results are dispatched to the Info>s on this class. Callers can check
/// to find out the current status.
///
/// The (optional) type name of the single test class to run
/// Set to true to enable diagnostic messages; set to false to disable them.
/// By default, uses the value from the assembly configuration file.
/// Set to choose the default display name style for test methods.
/// By default, uses the value from the assembly configuration file. (This parameter is ignored for xUnit.net v1 tests.)
/// Set to choose the default display name style options for test methods.
/// By default, uses the value from the assembly configuration file. (This parameter is ignored for xUnit.net v1 tests.)
/// Set to true to pre-enumerate individual theory tests; set to false to use
/// a single test case for the theory. By default, uses the value from the assembly configuration file. (This parameter is ignored
/// for xUnit.net v1 tests.)
/// Set to true to run test collections in parallel; set to false to run them sequentially.
/// By default, uses the value from the assembly configuration file. (This parameter is ignored for xUnit.net v1 tests.)
/// Set to 0 to use unlimited threads; set to any other positive integer to limit to an exact number
/// of threads. By default, uses the value from the assembly configuration file. (This parameter is ignored for xUnit.net v1 tests.)
/// Set to true to enable internal diagnostic messages; set to false to disable them.
/// By default, uses the value from the assembly configuration file.
public void Discover(
string typeName = null,
bool? diagnosticMessages = null,
TestMethodDisplay? methodDisplay = null,
TestMethodDisplayOptions? methodDisplayOptions = null,
bool? preEnumerateTheories = null,
bool? internalDiagnosticMessages = null)
{
cancelled = false;
testCasesDiscovered = 0;
testCasesToRun.Clear();
XunitWorkerThread.QueueUserWorkItem(() =>
{
var discoveryOptions = GetDiscoveryOptions(diagnosticMessages, methodDisplay, methodDisplayOptions, preEnumerateTheories, internalDiagnosticMessages);
if (typeName != null)
controller.Find(typeName, false, this, discoveryOptions);
else
controller.Find(false, this, discoveryOptions);
if (cancelled)
{
// Synthesize the execution complete message, since we're not going to run at all
if (OnExecutionComplete != null)
OnExecutionComplete(ExecutionCompleteInfo.Empty);
return;
}
});
}
public void Run(List cases,
bool? diagnosticMessages = null,
bool? parallel = null,
int? maxParallelThreads = null,
bool? internalDiagnosticMessages = null)
{
cancelled = false;
testCasesDiscovered = cases.Count();
testCasesToRun.Clear();
testCasesToRun.AddRange(cases);
XunitWorkerThread.QueueUserWorkItem(() =>
{
if (cancelled)
{
// Synthesize the execution complete message, since we're not going to run at all
if (OnExecutionComplete != null)
OnExecutionComplete(ExecutionCompleteInfo.Empty);
return;
}
var executionOptions = GetExecutionOptions(diagnosticMessages, parallel, maxParallelThreads, internalDiagnosticMessages);
controller.RunTests(testCasesToRun, this, executionOptions);
});
}
///
/// Creates an assembly runner that discovers and run tests in a separate app domain.
///
/// The test assembly.
/// The test assembly configuration file.
/// If set to true, runs tests in a shadow copied app domain, which allows
/// tests to be discovered and run without locking assembly files on disk.
/// The path on disk to use for shadow copying; if null, a folder
/// will be automatically (randomly) generated
//public static AssemblyRunner2 WithAppDomain(string assemblyFileName,
// string configFileName = null,
// bool shadowCopy = true,
// string shadowCopyFolder = null)
//{
// //Guard.ArgumentValid(nameof(shadowCopyFolder), "Cannot set shadowCopyFolder if shadowCopy is false", shadowCopy == true || shadowCopyFolder == null);
// return new AssemblyRunner2(AppDomainSupport.Required, assemblyFileName, configFileName, shadowCopy, shadowCopyFolder);
//}
///
/// Creates an assembly runner that discovers and runs tests without a separate app domain.
///
/// The test assembly.
public static AssemblyRunner2 WithoutAppDomain(string assemblyFileName, TaskCompletionSource tcs, CancellationToken cancellationToken)
{
return new AssemblyRunner2(AppDomainSupport.Denied, assemblyFileName, tcs, cancellationToken);
}
bool DispatchMessage(IMessageSinkMessage message, HashSet messageTypes, Action handler)
where TMessage : class
{
if (messageTypes == null || !MessageTypeNames.TryGetValue(typeof(TMessage), out var typeName) || !messageTypes.Contains(typeName))
return false;
handler((TMessage)message);
return true;
}
bool IMessageSinkWithTypes.OnMessageWithTypes(IMessageSinkMessage message, HashSet messageTypes)
{
if (cancellationToken.IsCancellationRequested)
{
cancelled = true;
}
if (DispatchMessage(message, messageTypes, testDiscovered =>
{
++testCasesDiscovered;
if (TestCaseFilter == null || TestCaseFilter(testDiscovered.TestCase))
testCasesToRun.Add(testDiscovered.TestCase);
}))
return !cancelled;
if (DispatchMessage(message, messageTypes, discoveryComplete =>
{
OnDiscoveryComplete?.Invoke(new TestDiscoveryInfo(testCasesToRun, assemblyFileName));
tcs.TrySetResult("");
}))
return !cancelled;
if (DispatchMessage(message, messageTypes, assemblyFinished =>
{
OnExecutionComplete?.Invoke(new ExecutionCompleteInfo(assemblyFinished.TestsRun, assemblyFinished.TestsFailed, assemblyFinished.TestsSkipped, assemblyFinished.ExecutionTime));
tcs.TrySetResult("");
}))
return !cancelled;
if (OnDiagnosticMessage != null)
if (DispatchMessage(message, messageTypes, m => OnDiagnosticMessage(new DiagnosticMessageInfo(m.Message))))
return !cancelled;
if (OnTestFailed != null)
if (DispatchMessage(message, messageTypes, m => OnTestFailed(new TestFailedInfo(m.TestClass.Class.Name, m.TestMethod.Method.Name, m.TestCase.Traits, m.Test.DisplayName, m.TestCollection.DisplayName, m.ExecutionTime, m.Output, m.ExceptionTypes.FirstOrDefault(), m.Messages.FirstOrDefault(), m.StackTraces.FirstOrDefault()))))
return !cancelled;
if (OnTestFinished != null)
if (DispatchMessage(message, messageTypes, m => OnTestFinished(new TestFinishedInfo(m.TestClass.Class.Name, m.TestMethod.Method.Name, m.TestCase.Traits, m.Test.DisplayName, m.TestCollection.DisplayName, m.ExecutionTime, m.Output))))
return !cancelled;
if (OnTestOutput != null)
if (DispatchMessage(message, messageTypes, m => OnTestOutput(new TestOutputInfo(m.TestClass.Class.Name, m.TestMethod.Method.Name, m.TestCase.Traits, m.Test.DisplayName, m.TestCollection.DisplayName, m.Output))))
return !cancelled;
if (OnTestPassed != null)
if (DispatchMessage(message, messageTypes, m => OnTestPassed(new TestPassedInfo(m.TestClass.Class.Name, m.TestMethod.Method.Name, m.TestCase.Traits, m.Test.DisplayName, m.TestCollection.DisplayName, m.ExecutionTime, m.Output))))
return !cancelled;
if (OnTestSkipped != null)
if (DispatchMessage(message, messageTypes, m => OnTestSkipped(new TestSkippedInfo(m.TestClass.Class.Name, m.TestMethod.Method.Name, m.TestCase.Traits, m.Test.DisplayName, m.TestCollection.DisplayName, m.Reason))))
return !cancelled;
if (OnTestStarting != null)
if (DispatchMessage(message, messageTypes, m => OnTestStarting(new TestStartingInfo(m.TestClass.Class.Name, m.TestMethod.Method.Name, m.TestCase.Traits, m.Test.DisplayName, m.TestCollection.DisplayName))))
return !cancelled;
if (OnErrorMessage != null)
{
if (DispatchMessage(message, messageTypes, m => OnErrorMessage(new ErrorMessageInfo(ErrorMessageType.CatastrophicError, m.ExceptionTypes.FirstOrDefault(), m.Messages.FirstOrDefault(), m.StackTraces.FirstOrDefault()))))
return !cancelled;
if (DispatchMessage(message, messageTypes, m => OnErrorMessage(new ErrorMessageInfo(ErrorMessageType.TestAssemblyCleanupFailure, m.ExceptionTypes.FirstOrDefault(), m.Messages.FirstOrDefault(), m.StackTraces.FirstOrDefault()))))
return !cancelled;
if (DispatchMessage(message, messageTypes, m => OnErrorMessage(new ErrorMessageInfo(ErrorMessageType.TestCaseCleanupFailure, m.ExceptionTypes.FirstOrDefault(), m.Messages.FirstOrDefault(), m.StackTraces.FirstOrDefault()))))
return !cancelled;
if (DispatchMessage(message, messageTypes, m => OnErrorMessage(new ErrorMessageInfo(ErrorMessageType.TestClassCleanupFailure, m.ExceptionTypes.FirstOrDefault(), m.Messages.FirstOrDefault(), m.StackTraces.FirstOrDefault()))))
return !cancelled;
if (DispatchMessage(message, messageTypes, m => OnErrorMessage(new ErrorMessageInfo(ErrorMessageType.TestCleanupFailure, m.ExceptionTypes.FirstOrDefault(), m.Messages.FirstOrDefault(), m.StackTraces.FirstOrDefault()))))
return !cancelled;
if (DispatchMessage(message, messageTypes, m => OnErrorMessage(new ErrorMessageInfo(ErrorMessageType.TestCollectionCleanupFailure, m.ExceptionTypes.FirstOrDefault(), m.Messages.FirstOrDefault(), m.StackTraces.FirstOrDefault()))))
return !cancelled;
if (DispatchMessage(message, messageTypes, m => OnErrorMessage(new ErrorMessageInfo(ErrorMessageType.TestMethodCleanupFailure, m.ExceptionTypes.FirstOrDefault(), m.Messages.FirstOrDefault(), m.StackTraces.FirstOrDefault()))))
return !cancelled;
}
return !cancelled;
}
}
}