feat: Matrix integration tests

.github/actions/setup-dotnet/action.yml

@@ -5,6 +5,10 @@ inputs:
     description: 'Whether to run dotnet build after restore'
     required: false
     default: 'true'
+  configuration:
+    description: 'What configuration to run (e.g., Release or Debug)'
+    required: false
+    default: 'Release'
 runs:
   using: 'composite'
   steps:
@@ -31,4 +35,4 @@ runs:
     - name: Build with dotnet
       if: inputs.build == 'true'
       shell: bash
-      run: dotnet build -p:ContinuousIntegrationBuild=True --configuration Release --no-restore
+      run: dotnet build -p:ContinuousIntegrationBuild=True --configuration ${{ inputs.configuration }} --no-restore

.github/workflows/Build-Test-And-Deploy.yaml

@@ -35,17 +35,17 @@ jobs:
 
       - name: Run .NET Tests
         id: run-dotnet-tests
-        run: dotnet test --no-build --configuration Release --blame-hang-timeout 15m --blame-hang-dump-type full -l trx --results-directory ./TestResults
+        run: dotnet test --no-build --configuration Release --blame-hang-timeout 15m --blame-hang-dump-type full -l trx --results-directory ./TestResults --filter TestType!=Integration
         env:
           POCKETLOGGER_LOG_PATH: ${{ github.workspace }}/artifacts/logs/pocketlogger.log
 
       - name: Convert TRX to Playlist
         if: ${{ steps.run-dotnet-tests.outcome == 'failure' || failure() }}
         uses: BenjaminMichaelis/trx-to-vsplaylist@v3
         with:
-          trx-file-path: './TestResults/*.trx'
-          test-outcomes: 'Failed'
-          artifact-name: 'linux-test-playlists'
+          trx-file-path: "./TestResults/*.trx"
+          test-outcomes: "Failed"
+          artifact-name: "linux-test-playlists"
 
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
@@ -93,24 +93,33 @@ jobs:
 
       - name: Run .NET Tests
         id: run-dotnet-tests
-        run: dotnet test --no-build --configuration Release --blame-hang-timeout 15m --blame-hang-dump-type full -l trx --results-directory ./TestResults
+        run: dotnet test --no-build --configuration Release --blame-hang-timeout 15m --blame-hang-dump-type full -l trx --results-directory ./TestResults --filter TestType!=Integration
         env:
           POCKETLOGGER_LOG_PATH: ${{ github.workspace }}/artifacts/logs/pocketlogger.log
 
       - name: Convert TRX to Playlist
         if: ${{ steps.run-dotnet-tests.outcome == 'failure' || failure() }}
         uses: BenjaminMichaelis/trx-to-vsplaylist@v3
         with:
-          trx-file-path: './TestResults/*.trx'
-          test-outcomes: 'Failed'
-          artifact-name: 'windows-test-playlists'
+          trx-file-path: "./TestResults/*.trx"
+          test-outcomes: "Failed"
+          artifact-name: "windows-test-playlists"
 
   integration-tests:
-    # Integration tests use Playwright and are gated behind RunIntegrationTests=true.
-    # IntegrationTestFactAttribute explicitly skips these tests on Linux, so a
-    # Windows runner is required to actually execute them.
-    runs-on: windows-latest
-
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [windows-latest, ubuntu-latest, macos-latest]
+        browser: [chromium, firefox, webkit]
+        exclude:
+          - os: windows-latest
+            browser: webkit
+          - os: ubuntu-latest
+            browser: webkit
+          - os: macos-latest
+            browser: firefox
+
+    runs-on: ${{ matrix.os }}
     steps:
       - uses: actions/checkout@v6
 
@@ -119,22 +128,25 @@ jobs:
 
       - name: Set up .NET environment
         uses: ./.github/actions/setup-dotnet
+        with:
+          configuration: Debug
 
       - name: Run .NET Integration Tests
         id: run-dotnet-integration-tests
-        run: dotnet test --no-build --configuration Release --blame-hang-timeout 15m --blame-hang-dump-type full -l trx --results-directory ./TestResults
+        run: dotnet test --no-build --configuration Debug --blame-hang-timeout 15m --blame-hang-dump-type full -l trx --results-directory ./TestResults --filter TestType=Integration -- Playwright.BrowserName=${{ matrix.browser }}
         env:
           RunIntegrationTests: true
           POCKETLOGGER_LOG_PATH: ${{ github.workspace }}/artifacts/logs/pocketlogger.log
           TRYDOTNET_PREBUILDS_PATH: ${{ github.workspace }}/artifacts/trydotnet-prebuilds
+          Playwright_BrowserName: ${{ matrix.browser }}
 
       - name: Convert TRX to Playlist
         if: ${{ steps.run-dotnet-integration-tests.outcome == 'failure' || failure() }}
         uses: BenjaminMichaelis/trx-to-vsplaylist@v3
         with:
-          trx-file-path: './TestResults/*.trx'
-          test-outcomes: 'Failed'
-          artifact-name: 'integration-test-playlists'
+          trx-file-path: "./TestResults/*.trx"
+          test-outcomes: "Failed"
+          artifact-name: "integration-test-playlists"
 
   deploy-development:
     if: github.event_name != 'pull_request_target' && github.event_name != 'pull_request'

src/Microsoft.TryDotNet.IntegrationTests/EditorTests.cs

@@ -15,6 +15,7 @@
 namespace Microsoft.TryDotNet.IntegrationTests;
 
 [LogToPocketLogger(FileNameEnvironmentVariable = "POCKETLOGGER_LOG_PATH")]
+[Trait("TestType", "Integration")]
 public class EditorTests : PlaywrightTestBase
 {
     public EditorTests(IntegratedServicesFixture services, ITestOutputHelper output) : base(services, output)

src/Microsoft.TryDotNet.IntegrationTests/IntegratedServicesFixture.cs

@@ -1,6 +1,7 @@
 // Copyright (c) .NET Foundation and contributors. All rights reserved.
 // Licensed under the MIT license. See LICENSE file in the project root for full license information.
 
+using System;
 using System.Threading;
 using System.Threading.Tasks;
 using Microsoft.DotNet.Interactive.CSharpProject.Build;

src/Microsoft.TryDotNet.IntegrationTests/IntegrationTestFactAttribute.cs

@@ -1,33 +1,30 @@
-// Copyright (c) .NET Foundation and contributors. All rights reserved.
-// Licensed under the MIT license. See LICENSE file in the project root for full license information.
-
-using System;
-using System.Runtime.InteropServices;
-
-using Xunit;
-
-namespace Microsoft.TryDotNet.IntegrationTests
-{
-    internal class IntegrationTestFactAttribute : FactAttribute
-    {
-        private const string EnvironmentVariableName = "RunIntegrationTests";
-
-        public IntegrationTestFactAttribute(string? skipReason = null)
-        {
-            var variableValue = Environment.GetEnvironmentVariable(EnvironmentVariableName) ?? "false";
-            switch (variableValue.ToLowerInvariant())
-            {
-                case "1":
-                case "true":
-                    if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux))
-                    {
-                        Skip = string.IsNullOrWhiteSpace(skipReason) ? "Ignored on Linux" : skipReason;
-                    }
-                    break;
-                default:
-                    Skip = $"Skipping integration tests because environment variable '{EnvironmentVariableName}' was not 'true' or '1'.";
-                    break;
-            }
-        }
-    }
-}
+// Copyright (c) .NET Foundation and contributors. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+using System;
+using System.Runtime.InteropServices;
+
+using Xunit;
+
+namespace Microsoft.TryDotNet.IntegrationTests
+{
+    internal class IntegrationTestFactAttribute : FactAttribute
+    {
+        private const string EnvironmentVariableName = "RunIntegrationTests";
+
+        public IntegrationTestFactAttribute(string? skipReason = null)
+        {
+            var variableValue = Environment.GetEnvironmentVariable(EnvironmentVariableName) ?? "false";
+            switch (variableValue.ToLowerInvariant())
+            {
+                case "1":
+                case "true":
+                    // Run tests
+                    break;
+                default:
+                    Skip = $"Skipping integration tests because environment variable '{EnvironmentVariableName}' was not 'true' or '1'.";
+                    break;
+            }
+        }
+    }
+}

src/Microsoft.TryDotNet.IntegrationTests/Microsoft.TryDotNet.IntegrationTests.csproj

@@ -10,6 +10,10 @@
     <DebugType>embedded</DebugType>
   </PropertyGroup>
 
+  <ItemGroup>
+    <Using Include="Xunit" />
+  </ItemGroup>
+
   <ItemGroup>
     <PackageReference Include="Assent" />
     <PackageReference Include="AwesomeAssertions" />
@@ -71,6 +75,5 @@ public static class BuildProperties
   <Target Name="PublishTool" BeforeTargets="CoreCompile" Condition="'$(DisableArcade)' != '1'">
     <Exec Command="dotnet publish -o $(PublishLocation)" WorkingDirectory="$(MSBuildThisFileDirectory)..\Microsoft.TryDotNet" />
   </Target>
-
 </Project>
 

src/Microsoft.TryDotNet.IntegrationTests/PlaywrightSession.cs

@@ -20,9 +20,24 @@ public PlaywrightSession(IPlaywright playwright, IBrowser browser)
 
     public IBrowser Browser { get; }
 
-    public static async Task<PlaywrightSession> StartAsync()
+    public static async Task<PlaywrightSession> StartAsync(string? browserName = null)
     {
-        var exitCode = Playwright.Program.Main(["install", "chromium"]);
+
+        string? selectedBrowser = browserName
+            ?? Environment.GetEnvironmentVariable("Playwright.BrowserName")
+            ?? Environment.GetEnvironmentVariable("Playwright_BrowserName");
+
+        int exitCode;
+        if (!string.IsNullOrWhiteSpace(selectedBrowser))
+        {
+            selectedBrowser = selectedBrowser.ToLowerInvariant();
+            exitCode = Playwright.Program.Main(["install", selectedBrowser]);
+        }
+        else
+        {
+            exitCode = Playwright.Program.Main(["install"]);
+            selectedBrowser = "chromium";
+        }
         if (exitCode is not 0)
         {
             throw new Exception($"Playwright exited with code {exitCode}");
@@ -37,7 +52,13 @@ public static async Task<PlaywrightSession> StartAsync()
             browserTypeLaunchOptions.Headless = false;
         }
 
-        var browser = await session.Chromium.LaunchAsync(browserTypeLaunchOptions).Timeout(TimeSpan.FromMinutes(5), "Timeout launching browser");
+        IBrowser browser = selectedBrowser switch
+        {
+            "chromium" => await session.Chromium.LaunchAsync(browserTypeLaunchOptions).Timeout(TimeSpan.FromMinutes(5), "Timeout launching browser"),
+            "firefox" => await session.Firefox.LaunchAsync(browserTypeLaunchOptions).Timeout(TimeSpan.FromMinutes(5), "Timeout launching browser"),
+            "webkit" => await session.Webkit.LaunchAsync(browserTypeLaunchOptions).Timeout(TimeSpan.FromMinutes(5), "Timeout launching browser"),
+            _ => throw new ArgumentException($"Unknown browser '{selectedBrowser}'. Valid values: chromium, firefox, webkit.")
+        };
 
         return new PlaywrightSession(session, browser);
     }

src/Microsoft.TryDotNet.IntegrationTests/TryDotNetJsIntegrationTests.cs

@@ -14,6 +14,7 @@
 namespace Microsoft.TryDotNet.IntegrationTests;
 
 [LogToPocketLogger(FileNameEnvironmentVariable = "POCKETLOGGER_LOG_PATH")]
+[Trait("TestType", "Integration")]
 public class TryDotNetJsIntegrationTests : PlaywrightTestBase
 {
     public TryDotNetJsIntegrationTests(IntegratedServicesFixture services, ITestOutputHelper output) : base(services, output)

src/Microsoft.TryDotNet.IntegrationTests/WasmRunnerTests.cs

@@ -12,6 +12,7 @@
 namespace Microsoft.TryDotNet.IntegrationTests;
 
 [LogToPocketLogger(FileNameEnvironmentVariable = "POCKETLOGGER_LOG_PATH")]
+[Trait("TestType", "Integration")]
 public class WasmRunnerTests : PlaywrightTestBase
 {
     public WasmRunnerTests(IntegratedServicesFixture services, ITestOutputHelper output) : base(services, output)