using System; using System.Collections.Generic; using System.Dynamic; using System.Linq; using System.Threading; using System.Threading.Tasks; using Xunit.Abstractions; using Xunit.Runner.Wpf; using Xunit.Sdk; namespace Xunit.Runners { /// /// A class which makes it simpler for casual runner authors to find and run tests and get results. /// Adapted from: https://github.com/xunit/xunit/blob/main/src/xunit.v3.runner.utility/Runners/AssemblyRunner.cs /// public class AssemblyRunner2 : LongLivedMarshalByRefObject, IDisposable, IMessageSinkWithTypes { static readonly Dictionary MessageTypeNames; private TaskCompletionSource tcs { get; set; } private CancellationToken cancellationToken { get; set; } volatile bool cancelled; bool disposed; readonly TestAssemblyConfiguration configuration; readonly IFrontController controller; string assemblyFileName { get; set; } int testCasesDiscovered; readonly List testCasesToRun = new List(); static AssemblyRunner2() { MessageTypeNames = new Dictionary(); AddMessageTypeName(); AddMessageTypeName(); AddMessageTypeName(); AddMessageTypeName(); AddMessageTypeName(); AddMessageTypeName(); AddMessageTypeName(); AddMessageTypeName(); AddMessageTypeName(); AddMessageTypeName(); AddMessageTypeName(); AddMessageTypeName(); AddMessageTypeName(); AddMessageTypeName(); AddMessageTypeName(); AddMessageTypeName(); AddMessageTypeName(); } AssemblyRunner2(AppDomainSupport appDomainSupport, string assemblyFileName, TaskCompletionSource tcs, CancellationToken cancellationToken, string configFileName = null, bool shadowCopy = true, string shadowCopyFolder = null) { this.tcs = tcs; this.cancellationToken = cancellationToken; this.assemblyFileName = assemblyFileName; controller = new XunitFrontController(appDomainSupport, assemblyFileName, configFileName, shadowCopy, shadowCopyFolder, diagnosticMessageSink: MessageSinkAdapter.Wrap(this)); configuration = ConfigReader.Load(assemblyFileName, configFileName); } /// /// Set to get notification of diagnostic messages. /// public Action OnDiagnosticMessage { get; set; } /// /// Set to get notification of when test discovery is complete. /// public Action OnDiscoveryComplete { get; set; } /// /// Set to get notification of error messages (unhandled exceptions outside of tests). /// public Action OnErrorMessage { get; set; } /// /// Set to get notification of when test execution is complete. /// public Action OnExecutionComplete { get; set; } /// /// Set to get notification of failed tests. /// public Action OnTestFailed { get; set; } /// /// Set to get notification of finished tests (regardless of outcome). /// public Action OnTestFinished { get; set; } /// /// Set to get real-time notification of test output (for xUnit.net v2 tests only). /// Note that output is captured and reported back to all the test completion Info>s /// in addition to being sent to this Info>. /// public Action OnTestOutput { get; set; } /// /// Set to get notification of passing tests. /// public Action OnTestPassed { get; set; } /// /// Set to get notification of skipped tests. /// public Action OnTestSkipped { get; set; } /// /// Set to get notification of when tests start running. /// public Action OnTestStarting { get; set; } /// /// Set to be able to filter the test cases to decide which ones to run. If this is not set, /// then all test cases will be run. /// public Func TestCaseFilter { get; set; } static void AddMessageTypeName() => MessageTypeNames.Add(typeof(T), typeof(T).FullName); /// /// Call to request that the current run be cancelled. Note that cancellation may not be /// instantaneous, and even after cancellation has been acknowledged, you can expect to /// receive all the cleanup-related messages. /// public void Cancel() { cancelled = true; } /// public void Dispose() { if (disposed) return; disposed = true; controller?.Dispose(); } ITestFrameworkDiscoveryOptions GetDiscoveryOptions(bool? diagnosticMessages, TestMethodDisplay? methodDisplay, TestMethodDisplayOptions? methodDisplayOptions, bool? preEnumerateTheories, bool? internalDiagnosticMessages) { var discoveryOptions = TestFrameworkOptions.ForDiscovery(configuration); discoveryOptions.SetSynchronousMessageReporting(true); if (diagnosticMessages.HasValue) discoveryOptions.SetDiagnosticMessages(diagnosticMessages); if (internalDiagnosticMessages.HasValue) discoveryOptions.SetDiagnosticMessages(internalDiagnosticMessages); if (methodDisplay.HasValue) discoveryOptions.SetMethodDisplay(methodDisplay); if (methodDisplayOptions.HasValue) discoveryOptions.SetMethodDisplayOptions(methodDisplayOptions); if (preEnumerateTheories.HasValue) discoveryOptions.SetPreEnumerateTheories(preEnumerateTheories); return discoveryOptions; } ITestFrameworkExecutionOptions GetExecutionOptions(bool? diagnosticMessages, bool? parallel, int? maxParallelThreads, bool? internalDiagnosticMessages) { var executionOptions = TestFrameworkOptions.ForExecution(configuration); executionOptions.SetSynchronousMessageReporting(true); if (diagnosticMessages.HasValue) executionOptions.SetDiagnosticMessages(diagnosticMessages); if (internalDiagnosticMessages.HasValue) executionOptions.SetDiagnosticMessages(internalDiagnosticMessages); if (parallel.HasValue) executionOptions.SetDisableParallelization(!parallel.GetValueOrDefault()); if (maxParallelThreads.HasValue) executionOptions.SetMaxParallelThreads(maxParallelThreads); return executionOptions; } /// /// Starts running tests from a single type (if provided) or the whole assembly (if not). This call returns /// immediately, and status results are dispatched to the Info>s on this class. Callers can check /// to find out the current status. /// /// The (optional) type name of the single test class to run /// Set to true to enable diagnostic messages; set to false to disable them. /// By default, uses the value from the assembly configuration file. /// Set to choose the default display name style for test methods. /// By default, uses the value from the assembly configuration file. (This parameter is ignored for xUnit.net v1 tests.) /// Set to choose the default display name style options for test methods. /// By default, uses the value from the assembly configuration file. (This parameter is ignored for xUnit.net v1 tests.) /// Set to true to pre-enumerate individual theory tests; set to false to use /// a single test case for the theory. By default, uses the value from the assembly configuration file. (This parameter is ignored /// for xUnit.net v1 tests.) /// Set to true to run test collections in parallel; set to false to run them sequentially. /// By default, uses the value from the assembly configuration file. (This parameter is ignored for xUnit.net v1 tests.) /// Set to 0 to use unlimited threads; set to any other positive integer to limit to an exact number /// of threads. By default, uses the value from the assembly configuration file. (This parameter is ignored for xUnit.net v1 tests.) /// Set to true to enable internal diagnostic messages; set to false to disable them. /// By default, uses the value from the assembly configuration file. public void Discover( string typeName = null, bool? diagnosticMessages = null, TestMethodDisplay? methodDisplay = null, TestMethodDisplayOptions? methodDisplayOptions = null, bool? preEnumerateTheories = null, bool? internalDiagnosticMessages = null) { cancelled = false; testCasesDiscovered = 0; testCasesToRun.Clear(); XunitWorkerThread.QueueUserWorkItem(() => { var discoveryOptions = GetDiscoveryOptions(diagnosticMessages, methodDisplay, methodDisplayOptions, preEnumerateTheories, internalDiagnosticMessages); if (typeName != null) controller.Find(typeName, false, this, discoveryOptions); else controller.Find(false, this, discoveryOptions); if (cancelled) { // Synthesize the execution complete message, since we're not going to run at all if (OnExecutionComplete != null) OnExecutionComplete(ExecutionCompleteInfo.Empty); return; } }); } public void Run(List cases, bool? diagnosticMessages = null, bool? parallel = null, int? maxParallelThreads = null, bool? internalDiagnosticMessages = null) { cancelled = false; testCasesDiscovered = cases.Count(); testCasesToRun.Clear(); testCasesToRun.AddRange(cases); XunitWorkerThread.QueueUserWorkItem(() => { if (cancelled) { // Synthesize the execution complete message, since we're not going to run at all if (OnExecutionComplete != null) OnExecutionComplete(ExecutionCompleteInfo.Empty); return; } var executionOptions = GetExecutionOptions(diagnosticMessages, parallel, maxParallelThreads, internalDiagnosticMessages); controller.RunTests(testCasesToRun, this, executionOptions); }); } /// /// Creates an assembly runner that discovers and run tests in a separate app domain. /// /// The test assembly. /// The test assembly configuration file. /// If set to true, runs tests in a shadow copied app domain, which allows /// tests to be discovered and run without locking assembly files on disk. /// The path on disk to use for shadow copying; if null, a folder /// will be automatically (randomly) generated //public static AssemblyRunner2 WithAppDomain(string assemblyFileName, // string configFileName = null, // bool shadowCopy = true, // string shadowCopyFolder = null) //{ // //Guard.ArgumentValid(nameof(shadowCopyFolder), "Cannot set shadowCopyFolder if shadowCopy is false", shadowCopy == true || shadowCopyFolder == null); // return new AssemblyRunner2(AppDomainSupport.Required, assemblyFileName, configFileName, shadowCopy, shadowCopyFolder); //} /// /// Creates an assembly runner that discovers and runs tests without a separate app domain. /// /// The test assembly. public static AssemblyRunner2 WithoutAppDomain(string assemblyFileName, TaskCompletionSource tcs, CancellationToken cancellationToken) { return new AssemblyRunner2(AppDomainSupport.Denied, assemblyFileName, tcs, cancellationToken); } bool DispatchMessage(IMessageSinkMessage message, HashSet messageTypes, Action handler) where TMessage : class { if (messageTypes == null || !MessageTypeNames.TryGetValue(typeof(TMessage), out var typeName) || !messageTypes.Contains(typeName)) return false; handler((TMessage)message); return true; } bool IMessageSinkWithTypes.OnMessageWithTypes(IMessageSinkMessage message, HashSet messageTypes) { if (cancellationToken.IsCancellationRequested) { cancelled = true; } if (DispatchMessage(message, messageTypes, testDiscovered => { ++testCasesDiscovered; if (TestCaseFilter == null || TestCaseFilter(testDiscovered.TestCase)) testCasesToRun.Add(testDiscovered.TestCase); })) return !cancelled; if (DispatchMessage(message, messageTypes, discoveryComplete => { OnDiscoveryComplete?.Invoke(new TestDiscoveryInfo(testCasesToRun, assemblyFileName)); tcs.TrySetResult(""); })) return !cancelled; if (DispatchMessage(message, messageTypes, assemblyFinished => { OnExecutionComplete?.Invoke(new ExecutionCompleteInfo(assemblyFinished.TestsRun, assemblyFinished.TestsFailed, assemblyFinished.TestsSkipped, assemblyFinished.ExecutionTime)); tcs.TrySetResult(""); })) return !cancelled; if (OnDiagnosticMessage != null) if (DispatchMessage(message, messageTypes, m => OnDiagnosticMessage(new DiagnosticMessageInfo(m.Message)))) return !cancelled; if (OnTestFailed != null) if (DispatchMessage(message, messageTypes, m => OnTestFailed(new TestFailedInfo(m.TestClass.Class.Name, m.TestMethod.Method.Name, m.TestCase.Traits, m.Test.DisplayName, m.TestCollection.DisplayName, m.ExecutionTime, m.Output, m.ExceptionTypes.FirstOrDefault(), m.Messages.FirstOrDefault(), m.StackTraces.FirstOrDefault())))) return !cancelled; if (OnTestFinished != null) if (DispatchMessage(message, messageTypes, m => OnTestFinished(new TestFinishedInfo(m.TestClass.Class.Name, m.TestMethod.Method.Name, m.TestCase.Traits, m.Test.DisplayName, m.TestCollection.DisplayName, m.ExecutionTime, m.Output)))) return !cancelled; if (OnTestOutput != null) if (DispatchMessage(message, messageTypes, m => OnTestOutput(new TestOutputInfo(m.TestClass.Class.Name, m.TestMethod.Method.Name, m.TestCase.Traits, m.Test.DisplayName, m.TestCollection.DisplayName, m.Output)))) return !cancelled; if (OnTestPassed != null) if (DispatchMessage(message, messageTypes, m => OnTestPassed(new TestPassedInfo(m.TestClass.Class.Name, m.TestMethod.Method.Name, m.TestCase.Traits, m.Test.DisplayName, m.TestCollection.DisplayName, m.ExecutionTime, m.Output)))) return !cancelled; if (OnTestSkipped != null) if (DispatchMessage(message, messageTypes, m => OnTestSkipped(new TestSkippedInfo(m.TestClass.Class.Name, m.TestMethod.Method.Name, m.TestCase.Traits, m.Test.DisplayName, m.TestCollection.DisplayName, m.Reason)))) return !cancelled; if (OnTestStarting != null) if (DispatchMessage(message, messageTypes, m => OnTestStarting(new TestStartingInfo(m.TestClass.Class.Name, m.TestMethod.Method.Name, m.TestCase.Traits, m.Test.DisplayName, m.TestCollection.DisplayName)))) return !cancelled; if (OnErrorMessage != null) { if (DispatchMessage(message, messageTypes, m => OnErrorMessage(new ErrorMessageInfo(ErrorMessageType.CatastrophicError, m.ExceptionTypes.FirstOrDefault(), m.Messages.FirstOrDefault(), m.StackTraces.FirstOrDefault())))) return !cancelled; if (DispatchMessage(message, messageTypes, m => OnErrorMessage(new ErrorMessageInfo(ErrorMessageType.TestAssemblyCleanupFailure, m.ExceptionTypes.FirstOrDefault(), m.Messages.FirstOrDefault(), m.StackTraces.FirstOrDefault())))) return !cancelled; if (DispatchMessage(message, messageTypes, m => OnErrorMessage(new ErrorMessageInfo(ErrorMessageType.TestCaseCleanupFailure, m.ExceptionTypes.FirstOrDefault(), m.Messages.FirstOrDefault(), m.StackTraces.FirstOrDefault())))) return !cancelled; if (DispatchMessage(message, messageTypes, m => OnErrorMessage(new ErrorMessageInfo(ErrorMessageType.TestClassCleanupFailure, m.ExceptionTypes.FirstOrDefault(), m.Messages.FirstOrDefault(), m.StackTraces.FirstOrDefault())))) return !cancelled; if (DispatchMessage(message, messageTypes, m => OnErrorMessage(new ErrorMessageInfo(ErrorMessageType.TestCleanupFailure, m.ExceptionTypes.FirstOrDefault(), m.Messages.FirstOrDefault(), m.StackTraces.FirstOrDefault())))) return !cancelled; if (DispatchMessage(message, messageTypes, m => OnErrorMessage(new ErrorMessageInfo(ErrorMessageType.TestCollectionCleanupFailure, m.ExceptionTypes.FirstOrDefault(), m.Messages.FirstOrDefault(), m.StackTraces.FirstOrDefault())))) return !cancelled; if (DispatchMessage(message, messageTypes, m => OnErrorMessage(new ErrorMessageInfo(ErrorMessageType.TestMethodCleanupFailure, m.ExceptionTypes.FirstOrDefault(), m.Messages.FirstOrDefault(), m.StackTraces.FirstOrDefault())))) return !cancelled; } return !cancelled; } } }