Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Refactor TestMethodRunner.Execute so that it returns TestResult (from TestFramework) rather than UnitTestResult (from TestAdapter) #4591

Merged
merged 5 commits into from
Jan 13, 2025
Merged

Refactor TestMethodRunner.Execute so that it returns TestResult (from TestFramework) rather than UnitTestResult (from TestAdapter) #4591

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
65ddc02
Refactor TestMethodRunner.Execute so that it returns TestResult (from…
Youssef1313 Jan 9, 2025
345b9a0
Fix build
Youssef1313 Jan 9, 2025
9f33522
Fix tests build
Youssef1313 Jan 9, 2025
ac10fd7
More extensive refactoring
Youssef1313 Jan 10, 2025
e2e50a7
Revert "More extensive refactoring"
Youssef1313 Jan 10, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
41 changes: 21 additions & 20 deletions src/Adapter/MSTest.TestAdapter/Execution/TestMethodRunner.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -62,15 +62,15 @@ public TestMethodRunner(TestMethodInfo testMethodInfo, TestMethod testMethod, IT
/// Executes a test.
/// </summary>
/// <returns>The test results.</returns>
internal UnitTestResult[] Execute(string initializationLogs, string initializationErrorLogs, string initializationTrace, string initializationTestContextMessages)
internal List<TestResult> Execute(string initializationLogs, string initializationErrorLogs, string initializationTrace, string initializationTestContextMessages)
{
bool isSTATestClass = AttributeComparer.IsDerived<STATestClassAttribute>(_testMethodInfo.Parent.ClassAttribute);
bool isSTATestMethod = AttributeComparer.IsDerived<STATestMethodAttribute>(_testMethodInfo.TestMethodOptions.Executor);
bool isSTARequested = isSTATestClass || isSTATestMethod;
bool isWindowsOS = RuntimeInformation.IsOSPlatform(OSPlatform.Windows);
if (isSTARequested && isWindowsOS && Thread.CurrentThread.GetApartmentState() != ApartmentState.STA)
{
UnitTestResult[] results = Array.Empty<UnitTestResult>();
List<TestResult>? results = null;
Thread entryPointThread = new(() => results = SafeRunTestMethod(initializationLogs, initializationErrorLogs, initializationTrace, initializationTestContextMessages))
{
Name = (isSTATestClass, isSTATestMethod) switch
Expand All @@ -93,7 +93,7 @@ internal UnitTestResult[] Execute(string initializationLogs, string initializati
PlatformServiceProvider.Instance.AdapterTraceLogger.LogError(ex.ToString());
}

return results;
return results ?? new();
}
else
{
Expand All @@ -107,42 +107,43 @@ internal UnitTestResult[] Execute(string initializationLogs, string initializati
}

// Local functions
UnitTestResult[] SafeRunTestMethod(string initializationLogs, string initializationErrorLogs, string initializationTrace, string initializationTestContextMessages)
List<TestResult> SafeRunTestMethod(string initializationLogs, string initializationErrorLogs, string initializationTrace, string initializationTestContextMessages)
{
UnitTestResult[]? result = null;
List<TestResult>? result = null;

try
{
result = RunTestMethod();
}
catch (TestFailedException ex)
{
result = [new UnitTestResult(ex)];
result = [new TestResult() { TestFailureException = ex }];
}
catch (Exception ex)
{
if (result == null || result.Length == 0)
if (result == null || result.Count == 0)
{
result = [new()];
result = [new TestResult() { Outcome = UTF.UnitTestOutcome.Error }];
}

#pragma warning disable IDE0056 // Use index operator
result[result.Length - 1] = new UnitTestResult(new TestFailedException(UnitTestOutcome.Error, ex.TryGetMessage(), ex.TryGetStackTraceInformation()))
result[result.Count - 1] = new TestResult()
{
StandardOut = result[result.Length - 1].StandardOut,
StandardError = result[result.Length - 1].StandardError,
DebugTrace = result[result.Length - 1].DebugTrace,
TestContextMessages = result[result.Length - 1].TestContextMessages,
Duration = result[result.Length - 1].Duration,
TestFailureException = new TestFailedException(UnitTestOutcome.Error, ex.TryGetMessage(), ex.TryGetStackTraceInformation()),
LogOutput = result[result.Count - 1].LogOutput,
LogError = result[result.Count - 1].LogError,
DebugTrace = result[result.Count - 1].DebugTrace,
TestContextMessages = result[result.Count - 1].TestContextMessages,
Duration = result[result.Count - 1].Duration,
};
#pragma warning restore IDE0056 // Use index operator
}
finally
{
// Assembly initialize and class initialize logs are pre-pended to the first result.
UnitTestResult firstResult = result![0];
firstResult.StandardOut = initializationLogs + firstResult.StandardOut;
firstResult.StandardError = initializationErrorLogs + firstResult.StandardError;
TestResult firstResult = result![0];
firstResult.LogOutput = initializationLogs + firstResult.LogOutput;
firstResult.LogError = initializationErrorLogs + firstResult.LogError;
firstResult.DebugTrace = initializationTrace + firstResult.DebugTrace;
firstResult.TestContextMessages = initializationTestContextMessages + firstResult.TestContextMessages;
}
Expand All @@ -155,7 +156,7 @@ UnitTestResult[] SafeRunTestMethod(string initializationLogs, string initializat
/// Runs the test method.
/// </summary>
/// <returns>The test results.</returns>
internal UnitTestResult[] RunTestMethod()
internal List<TestResult> RunTestMethod()
{
DebugEx.Assert(_test != null, "Test should not be null.");
DebugEx.Assert(_testMethodInfo.TestMethod != null, "Test method should not be null.");
Expand All @@ -172,7 +173,7 @@ internal UnitTestResult[] RunTestMethod()
{
if (_test.TestDataSourceIgnoreMessage is not null)
{
return [new(UnitTestOutcome.Ignored, _test.TestDataSourceIgnoreMessage)];
return [new() { Outcome = UTF.UnitTestOutcome.Ignored, IgnoreReason = _test.TestDataSourceIgnoreMessage }];
}

object?[]? data = DataSerializationHelper.Deserialize(_test.SerializedData);
Expand Down Expand Up @@ -243,7 +244,7 @@ internal UnitTestResult[] RunTestMethod()
results.Add(emptyResult);
}

return results.ToUnitTestResults();
return results;
}

private bool TryExecuteDataSourceBasedTests(List<TestResult> results)
Expand Down
2 changes: 1 addition & 1 deletion src/Adapter/MSTest.TestAdapter/Execution/UnitTestRunner.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -180,7 +180,7 @@ internal UnitTestResult[] RunSingleTest(TestMethod testMethod, IDictionary<strin
// Run the test method
testContextForTestExecution.SetOutcome(testContextForClassInit.Context.CurrentTestOutcome);
var testMethodRunner = new TestMethodRunner(testMethodInfo, testMethod, testContextForTestExecution);
result = testMethodRunner.Execute(classInitializeResult.StandardOut!, classInitializeResult.StandardError!, classInitializeResult.DebugTrace!, classInitializeResult.TestContextMessages!);
result = testMethodRunner.Execute(classInitializeResult.StandardOut!, classInitializeResult.StandardError!, classInitializeResult.DebugTrace!, classInitializeResult.TestContextMessages!).ToUnitTestResults();
Evangelink marked this conversation as resolved.
Show resolved Hide resolved
}
}
}
Expand Down
37 changes: 19 additions & 18 deletions test/UnitTests/MSTestAdapter.UnitTests/Execution/TestMethodRunnerTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@

using Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter;
using Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.Execution;
using Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.Extensions;
using Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.Helpers;
using Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.ObjectModel;
using Microsoft.VisualStudio.TestPlatform.MSTestAdapter.PlatformServices;
Expand Down Expand Up @@ -82,7 +83,7 @@ public void ExecuteForTestThrowingExceptionShouldReturnUnitTestResultWithFailedO
var testMethodInfo = new TestableTestMethodInfo(_methodInfo, _testClassInfo, _testMethodOptions, () => throw new Exception("DummyException"));
var testMethodRunner = new TestMethodRunner(testMethodInfo, _testMethod, _testContextImplementation);

UnitTestResult[] results = testMethodRunner.Execute(string.Empty, string.Empty, string.Empty, string.Empty);
UnitTestResult[] results = testMethodRunner.Execute(string.Empty, string.Empty, string.Empty, string.Empty).ToUnitTestResults();
Verify(results[0].Outcome == AdapterTestOutcome.Failed);
Verify(results[0].ErrorMessage.Contains("Exception thrown while executing test"));
}
Expand All @@ -92,7 +93,7 @@ public void ExecuteForPassingTestShouldReturnUnitTestResultWithPassedOutcome()
var testMethodInfo = new TestableTestMethodInfo(_methodInfo, _testClassInfo, _testMethodOptions, () => new TestResult() { Outcome = UTF.UnitTestOutcome.Passed });
var testMethodRunner = new TestMethodRunner(testMethodInfo, _testMethod, _testContextImplementation);

UnitTestResult[] results = testMethodRunner.Execute(string.Empty, string.Empty, string.Empty, string.Empty);
UnitTestResult[] results = testMethodRunner.Execute(string.Empty, string.Empty, string.Empty, string.Empty).ToUnitTestResults();
Verify(results[0].Outcome == AdapterTestOutcome.Passed);
}

Expand All @@ -104,7 +105,7 @@ public void ExecuteShouldNotFillInDebugAndTraceLogsIfDebugTraceDisabled()
StringWriter writer = new(new StringBuilder("DummyTrace"));
_testablePlatformServiceProvider.MockTraceListener.Setup(tl => tl.GetWriter()).Returns(writer);

UnitTestResult[] results = testMethodRunner.Execute(string.Empty, string.Empty, string.Empty, string.Empty);
UnitTestResult[] results = testMethodRunner.Execute(string.Empty, string.Empty, string.Empty, string.Empty).ToUnitTestResults();
Verify(results[0].DebugTrace == string.Empty);
}

Expand All @@ -126,7 +127,7 @@ public void ExecuteShouldNotFillInDebugAndTraceLogsFromRunningTestMethod()
var testMethodRunner = new TestMethodRunner(testMethodInfo, _testMethod, _testContextImplementation);
_testablePlatformServiceProvider.MockTraceListener.Setup(tl => tl.GetWriter()).Returns(writer);

UnitTestResult[] results = testMethodRunner.Execute(string.Empty, string.Empty, string.Empty, string.Empty);
UnitTestResult[] results = testMethodRunner.Execute(string.Empty, string.Empty, string.Empty, string.Empty).ToUnitTestResults();

Verify(results[0].DebugTrace == string.Empty);
}
Expand All @@ -136,7 +137,7 @@ public void RunTestMethodForTestThrowingExceptionShouldReturnUnitTestResultWithF
var testMethodInfo = new TestableTestMethodInfo(_methodInfo, _testClassInfo, _testMethodOptions, () => throw new Exception("Dummy Exception"));
var testMethodRunner = new TestMethodRunner(testMethodInfo, _testMethod, _testContextImplementation);

UnitTestResult[] results = testMethodRunner.RunTestMethod();
UnitTestResult[] results = testMethodRunner.RunTestMethod().ToUnitTestResults();
Verify(results[0].Outcome == AdapterTestOutcome.Failed);
Verify(results[0].ErrorMessage.Contains("Exception thrown while executing test"));
}
Expand All @@ -155,7 +156,7 @@ public void RunTestMethodForMultipleResultsReturnMultipleResults()
var testMethodInfo = new TestableTestMethodInfo(_methodInfo, _testClassInfo, localTestMethodOptions, null);
var testMethodRunner = new TestMethodRunner(testMethodInfo, _testMethod, _testContextImplementation);

UnitTestResult[] results = testMethodRunner.Execute(string.Empty, string.Empty, string.Empty, string.Empty);
UnitTestResult[] results = testMethodRunner.Execute(string.Empty, string.Empty, string.Empty, string.Empty).ToUnitTestResults();
Verify(results.Length == 2);

Verify(results[0].Outcome == AdapterTestOutcome.Passed);
Expand All @@ -167,7 +168,7 @@ public void RunTestMethodForPassingTestThrowingExceptionShouldReturnUnitTestResu
var testMethodInfo = new TestableTestMethodInfo(_methodInfo, _testClassInfo, _testMethodOptions, () => new TestResult() { Outcome = UTF.UnitTestOutcome.Passed });
var testMethodRunner = new TestMethodRunner(testMethodInfo, _testMethod, _testContextImplementation);

UnitTestResult[] results = testMethodRunner.Execute(string.Empty, string.Empty, string.Empty, string.Empty);
UnitTestResult[] results = testMethodRunner.Execute(string.Empty, string.Empty, string.Empty, string.Empty).ToUnitTestResults();
Verify(results[0].Outcome == AdapterTestOutcome.Passed);
}

Expand All @@ -176,7 +177,7 @@ public void RunTestMethodForFailingTestThrowingExceptionShouldReturnUnitTestResu
var testMethodInfo = new TestableTestMethodInfo(_methodInfo, _testClassInfo, _testMethodOptions, () => new TestResult() { Outcome = UTF.UnitTestOutcome.Failed });
var testMethodRunner = new TestMethodRunner(testMethodInfo, _testMethod, _testContextImplementation);

UnitTestResult[] results = testMethodRunner.Execute(string.Empty, string.Empty, string.Empty, string.Empty);
UnitTestResult[] results = testMethodRunner.Execute(string.Empty, string.Empty, string.Empty, string.Empty).ToUnitTestResults();
Verify(results[0].Outcome == AdapterTestOutcome.Failed);
}

Expand All @@ -185,7 +186,7 @@ public void RunTestMethodShouldGiveTestResultAsPassedWhenTestMethodPasses()
var testMethodInfo = new TestableTestMethodInfo(_methodInfo, _testClassInfo, _testMethodOptions, () => new TestResult() { Outcome = UTF.UnitTestOutcome.Passed });
var testMethodRunner = new TestMethodRunner(testMethodInfo, _testMethod, _testContextImplementation);

UnitTestResult[] results = testMethodRunner.RunTestMethod();
UnitTestResult[] results = testMethodRunner.RunTestMethod().ToUnitTestResults();

// Since data is not provided, tests run normally giving passed as outcome.
Verify(results[0].Outcome == AdapterTestOutcome.Passed);
Expand All @@ -196,7 +197,7 @@ public void RunTestMethodShouldGiveTestResultAsFailedWhenTestMethodFails()
var testMethodInfo = new TestableTestMethodInfo(_methodInfo, _testClassInfo, _testMethodOptions, () => new TestResult() { Outcome = UTF.UnitTestOutcome.Failed });
var testMethodRunner = new TestMethodRunner(testMethodInfo, _testMethod, _testContextImplementation);

UnitTestResult[] results = testMethodRunner.RunTestMethod();
UnitTestResult[] results = testMethodRunner.RunTestMethod().ToUnitTestResults();

// Since data is not provided, tests run normally giving passed as outcome.
Verify(results[0].Outcome == AdapterTestOutcome.Failed);
Expand All @@ -217,7 +218,7 @@ public void RunTestMethodShouldRunDataDrivenTestsWhenDataIsProvidedUsingDataSour
_testablePlatformServiceProvider.MockReflectionOperations.Setup(rf => rf.GetCustomAttributes(_methodInfo, It.IsAny<bool>())).Returns(attributes);
_testablePlatformServiceProvider.MockTestDataSource.Setup(tds => tds.GetData(testMethodInfo, _testContextImplementation)).Returns([1, 2, 3]);

UnitTestResult[] results = testMethodRunner.RunTestMethod();
UnitTestResult[] results = testMethodRunner.RunTestMethod().ToUnitTestResults();

// check for outcome
Verify(results[0].Outcome == AdapterTestOutcome.Passed);
Expand Down Expand Up @@ -246,7 +247,7 @@ public void RunTestMethodShouldRunDataDrivenTestsWhenDataIsProvidedUsingDataRowA
// Setup mocks
_testablePlatformServiceProvider.MockReflectionOperations.Setup(ro => ro.GetCustomAttributes(_methodInfo, It.IsAny<Type>(), It.IsAny<bool>())).Returns(attributes);

UnitTestResult[] results = testMethodRunner.RunTestMethod();
UnitTestResult[] results = testMethodRunner.RunTestMethod().ToUnitTestResults();
Verify(results[0].Outcome == AdapterTestOutcome.Inconclusive);
}

Expand All @@ -263,7 +264,7 @@ public void RunTestMethodShouldSetDataRowIndexForDataDrivenTestsWhenDataIsProvid
_testablePlatformServiceProvider.MockReflectionOperations.Setup(rf => rf.GetCustomAttributes(_methodInfo, It.IsAny<bool>())).Returns(attributes);
_testablePlatformServiceProvider.MockTestDataSource.Setup(tds => tds.GetData(testMethodInfo, _testContextImplementation)).Returns([1, 2, 3]);

UnitTestResult[] results = testMethodRunner.RunTestMethod();
UnitTestResult[] results = testMethodRunner.RunTestMethod().ToUnitTestResults();

// check for datarowIndex
Verify(results[0].DatarowIndex == 0);
Expand All @@ -289,7 +290,7 @@ public void RunTestMethodShouldRunOnlyDataSourceTestsWhenBothDataSourceAndDataRo
_testablePlatformServiceProvider.MockReflectionOperations.Setup(rf => rf.GetCustomAttributes(_methodInfo, It.IsAny<bool>())).Returns(attributes);
_testablePlatformServiceProvider.MockTestDataSource.Setup(tds => tds.GetData(testMethodInfo, _testContextImplementation)).Returns([1, 2, 3]);

UnitTestResult[] results = testMethodRunner.RunTestMethod();
UnitTestResult[] results = testMethodRunner.RunTestMethod().ToUnitTestResults();

// check for datarowIndex as only DataSource Tests are Run
Verify(results[0].DatarowIndex == 0);
Expand All @@ -315,7 +316,7 @@ public void RunTestMethodShouldFillInDisplayNameWithDataRowDisplayNameIfProvided
// Setup mocks
_testablePlatformServiceProvider.MockReflectionOperations.Setup(ro => ro.GetCustomAttributes(_methodInfo, It.IsAny<bool>())).Returns(attributes);

UnitTestResult[] results = testMethodRunner.RunTestMethod();
UnitTestResult[] results = testMethodRunner.RunTestMethod().ToUnitTestResults();

Verify(results.Length == 1);
Verify(results[0].DisplayName == "DataRowTestDisplayName");
Expand All @@ -338,7 +339,7 @@ public void RunTestMethodShouldFillInDisplayNameWithDataRowArgumentsIfNoDisplayN
// Setup mocks
_testablePlatformServiceProvider.MockReflectionOperations.Setup(rf => rf.GetCustomAttributes(_methodInfo, It.IsAny<bool>())).Returns(attributes);

UnitTestResult[] results = testMethodRunner.RunTestMethod();
UnitTestResult[] results = testMethodRunner.RunTestMethod().ToUnitTestResults();

Verify(results.Length == 1);
Verify(results[0].DisplayName is "dummyTestName (2,\"DummyString\")" or "DummyTestMethod (2,DummyString)", $"Display name: {results[0].DisplayName}");
Expand All @@ -364,7 +365,7 @@ public void RunTestMethodShouldSetResultFilesIfPresentForDataDrivenTests()
// Setup mocks
_testablePlatformServiceProvider.MockReflectionOperations.Setup(rf => rf.GetCustomAttributes(_methodInfo, It.IsAny<bool>())).Returns(attributes);

UnitTestResult[] results = testMethodRunner.RunTestMethod();
UnitTestResult[] results = testMethodRunner.RunTestMethod().ToUnitTestResults();
Verify(results[0].ResultFiles.ToList().Contains("C:\\temp.txt"));
Verify(results[1].ResultFiles.ToList().Contains("C:\\temp.txt"));
}
Expand Down Expand Up @@ -404,7 +405,7 @@ private void RunTestMethodWithEmptyDataSourceShouldFailIfConsiderEmptyDataSource

if (considerEmptyAsInconclusive)
{
UnitTestResult[] results = testMethodRunner.RunTestMethod();
UnitTestResult[] results = testMethodRunner.RunTestMethod().ToUnitTestResults();
Verify(results[0].Outcome == AdapterTestOutcome.Inconclusive);
}
else
Expand Down
Loading