refactor(ci): Refactor CI to run integration tests as separate workflow (#413)

* Refactor CI to run integration tests as separate workflow

* Tool restore

* correct cache path

* conditionally use container registry

* use sln because net8

* fix typo

* Correct trait filter

* Correct mistake again

* fix again

* fml

* clarify names

* hopefully we're properly filtering test categories now

* maybe this?

* What does this do?

* revert is test project changes

* IsTestProject fix

* Correct test setup for automate

* maybe fix unit tests

* docker-compose-file alighment

* remove debug

* Ok tests should now pass

.github/workflows/integration-test.yml

@@ -0,0 +1,53 @@
+name: Integration Test
+
+on:
+  workflow_call:
+    inputs:
+      docker-compose-file:
+        required: true
+        type: string
+      use-github-container-registry:
+        default: false
+        type: boolean
+
+jobs:
+  integration-test:
+    env:
+      Solution: "Speckle.Sdk.sln"
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v5
+
+      - name: Setup .NET
+        uses: actions/setup-dotnet@v5
+        with:
+          dotnet-version: 8.x.x
+          cache: true
+          cache-dependency-path: "**/packages.lock.json"
+
+      - name: 🔐 Login to Github Container Registry
+        if: ${{ inputs.use-github-container-registry }}
+        uses: docker/login-action@v3
+        with:
+          registry: "ghcr.io"
+          username: ${{ github.actor }}
+          password: ${{ github.token }}
+
+      - name: ⚙️ Spin up Server
+        run: docker compose -f ${{ inputs.docker-compose-file }} up --wait
+
+      - name: 📦 Restore
+        run: dotnet restore ${{ env.Solution }} --locked-mode
+
+      - name: 🏗️ Build
+        run: dotnet build ${{ env.Solution }} --configuration Release --no-restore -warnaserror
+
+      - name: 🔨 Integration Tests
+        run: dotnet test ${{ env.Solution }} --filter "Category=Integration" --configuration Release --no-build --no-restore --verbosity=normal /p:AltCover=true /p:AltCoverAttributeFilter=ExcludeFromCodeCoverage
+
+      - name: Upload coverage reports to Codecov with GitHub Action
+        uses: codecov/codecov-action@v5
+        with:
+          files: tests/**/coverage.xml
+          token: ${{ secrets.CODECOV_TOKEN }}

.github/workflows/pr.yml

@@ -1,46 +1,52 @@
-name: .NET CI Build
+name: PR Test
 
 on:
   pull_request:
 
 jobs:
   build:
+    env:
+      Solution: "Speckle.Sdk.sln"
     runs-on: ubuntu-latest
     steps:
-    - name: Checkout
+      - name: Checkout
-      uses: actions/checkout@v5
+        uses: actions/checkout@v5
 
-    - name: Setup .NET
+      - name: Setup .NET
-      uses: actions/setup-dotnet@v5
+        uses: actions/setup-dotnet@v5
-      with:
+        with:
-        dotnet-version: 8.x.x
+          dotnet-version: 8.x.x
-        
+          cache: true
-    - uses: actions/cache@v4
+          cache-dependency-path: "**/packages.lock.json"
-      with:
+
-        path: ~/.nuget/packages
+      - name: 📦 Tool Restore
-        key: ${{ runner.os }}-nuget-${{ hashFiles('**/packages.lock.json') }}
+        run: dotnet tool restore
-        
+
-    - id: set-version
+      - name: 📄 Format
-      name: Set version to output
+        run: dotnet csharpier check .
-      run: |
+
-        SEMVER="3.0.99.${{ github.run_number }}"
+      - name: 📦 Restore
-        FILE_VERSION=$(echo "$SEMVER" | sed -E 's/^([0-9]+\.[0-9]+\.[0-9]+).*/\1/')
+        run: dotnet restore ${{ env.Solution }} --locked-mode
-        FILE_VERSION="$FILE_VERSION.${{ github.run_number }}"
+
-        
+      - name: 🏗️ Build
-        echo "semver=$SEMVER" >> "$GITHUB_OUTPUT"
+        run: dotnet build ${{ env.Solution }} --configuration Release --no-restore -warnaserror
-        echo "fileVersion=$FILE_VERSION" >> "$GITHUB_OUTPUT"
+
-        
+      - name: 🔨 Unit Tests
-        echo $SEMVER
+        run: dotnet test ${{ env.Solution }} --configuration Release --filter "Category!=Integration" --no-build --no-restore --verbosity=normal /p:AltCover=true /p:AltCoverAttributeFilter=ExcludeFromCodeCoverage
-        echo $FILE_VERSION
+
-        
+      - name: Upload coverage reports to Codecov with GitHub Action
-    - name: 🔫 Build All
+        uses: codecov/codecov-action@v5
-      run: ./build.sh
+        with:
-      env:
+          files: tests/**/coverage.xml
-        SEMVER: ${{ steps.set-version.outputs.SEMVER }}
+          token: ${{ secrets.CODECOV_TOKEN }}
-        FILE_VERSION: ${{ steps.set-version.outputs.FILE_VERSION }}
+
-      
+  integration-test-internal:
-    - name: Upload coverage reports to Codecov with GitHub Action
+    uses: "./.github/workflows/integration-test.yml"
-      uses: codecov/codecov-action@v5
+    with:
-      with:
+      docker-compose-file: "docker-compose-internal.yml"
-        files: tests/**/coverage.xml
+      use-github-container-registry: true
-        token: ${{ secrets.CODECOV_TOKEN }}
+
+  integration-test-public:
+    uses: "./.github/workflows/integration-test.yml"
+    with:
+      docker-compose-file: "docker-compose.yml"

.github/workflows/release.yml

@@ -8,9 +8,9 @@ jobs:
   build:
     runs-on: ubuntu-latest
     environment:
-      name: 'nuget.org'
+      name: "nuget.org"
     permissions:
-      id-token: write  # enable GitHub OIDC token issuance for this job
+      id-token: write # enable GitHub OIDC token issuance for this job
 
     steps:
       - name: Checkout
@@ -20,11 +20,8 @@ jobs:
         uses: actions/setup-dotnet@v5
         with:
           dotnet-version: 8.x.x
-
+          cache: true
-      - uses: actions/cache@v4
+          cache-dependency-path: "**/packages.lock.json"
-        with:
-          path: ~/.nuget/packages
-          key: ${{ runner.os }}-nuget-${{ hashFiles('**/packages.lock.json') }}
 
       - id: set-version
         name: Set version to output

Directory.Build.targets

@@ -1,5 +1,5 @@
 <Project>
-  <PropertyGroup Condition="'$(IsTestProject)' == 'true'">
+  <PropertyGroup Condition="'$(IsTestProject)' == 'true' or '$(TestProjectAnalyserRules)' == 'true' ">
     <NoWarn>
       <!-- Things we need to test -->
       CS0618;CA1034;CA2201;CA1051;CA1040;CA1724;CA1065;

README.md

@@ -65,10 +65,12 @@ Docs are a bit patchy [https://docs.speckle.systems/developers/looking-for-devel
 ### Tests
 
 There are several test projects. It is a requirement that all tests pass for PRs to be merged.
+
 The Integration test projects require a local server to be running.
+You must have docker installed. Then you can run `docker compose up` from the root of the repo to start the required containers.
 
-You must have docker installed. Then you can run `docker compose up --wait` from the root of the repo to start the required containers.
+In CI, they will be run against both the public and private versions of the server.
-
+It is important that we remain compatible with both server versions.
 ## Contributing
 
 Before embarking on submitting a patch, please make sure you read:

Speckle.Sdk.sln

@@ -27,6 +27,7 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "config", "config", "{DA2AED
 		CodeMetricsConfig.txt = CodeMetricsConfig.txt
 		Directory.Build.Targets = Directory.Build.Targets
 		.config\dotnet-tools.json = .config\dotnet-tools.json
+		docker-compose-internal.yml = docker-compose-internal.yml
 	EndProjectSection
 EndProject
 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "build", "build", "{58D37DA9-F948-48CA-9A73-F5BBBD533DBF}"

docker-compose-internal.yml

@@ -0,0 +1,118 @@
+name: "speckle-server"
+
+services:
+  ####
+  # Speckle Server dependencies
+  #######
+  postgres:
+    image: "postgres:16.4-alpine3.20@sha256:d898b0b78a2627cb4ee63464a14efc9d296884f1b28c841b0ab7d7c42f1fffdf"
+    restart: always
+    environment:
+      POSTGRES_DB: speckle
+      POSTGRES_USER: speckle
+      POSTGRES_PASSWORD: speckle
+    volumes:
+      - ./.volumes/postgres-data:/var/lib/postgresql/data/
+    healthcheck:
+      # the -U user has to match the POSTGRES_USER value
+      test: ["CMD-SHELL", "pg_isready -U speckle"]
+      interval: 5s
+      timeout: 5s
+      retries: 30
+
+  redis:
+    image: "valkey/valkey:8.1-alpine@sha256:0d27f0bca0249f61d060029a6aaf2e16b2c417d68d02a508e1dfb763fa2948b4"
+    restart: always
+    volumes:
+      - ./.volumes/redis-data:/data
+    healthcheck:
+      test: ["CMD", "redis-cli", "--raw", "incr", "ping"]
+      interval: 5s
+      timeout: 5s
+      retries: 30
+
+  minio:
+    image: "minio/minio:RELEASE.2023-10-25T06-33-25Z"
+    command: server /data --console-address ":9001"
+    restart: always
+    volumes:
+      - ./.volumes/minio-data:/data
+    ports:
+      - '127.0.0.1:9000:9000'
+      - '127.0.0.1:9001:9001'
+    healthcheck:
+      test:
+        [
+          "CMD-SHELL",
+          "curl -s -o /dev/null http://127.0.0.1:9000/minio/index.html",
+        ]
+      interval: 5s
+      timeout: 30s
+      retries: 30
+      start_period: 10s
+
+  speckle-server:
+    image: ghcr.io/specklesystems/speckle-server:latest
+    restart: always
+    healthcheck:
+      test:
+        - CMD
+        - /nodejs/bin/node
+        - -e
+        - "try { require('node:http').request({headers: {'Content-Type': 'application/json'}, port:3000, hostname:'127.0.0.1', path:'/readiness', method: 'GET', timeout: 2000 }, (res) => { body = ''; res.on('data', (chunk) => {body += chunk;}); res.on('end', () => {process.exit(Number(res.statusCode != 200 || body.toLowerCase().includes('error')));}); }).end(); } catch { process.exit(1); }"
+      interval: 10s
+      timeout: 10s
+      retries: 3
+      start_period: 90s
+    ports:
+      - "0.0.0.0:3000:3000"
+    depends_on:
+      postgres:
+        condition: service_healthy
+      redis:
+        condition: service_healthy
+      minio:
+        condition: service_healthy
+    environment:
+      # TODO: Change this to the URL of the speckle server, as accessed from the network
+      CANONICAL_URL: "http://127.0.0.1:8080"
+      SPECKLE_AUTOMATE_URL: "http://127.0.0.1:3030"
+      FRONTEND_ORIGIN: "http://127.0.0.1:8081"
+
+      # TODO: Change thvolumes:
+      REDIS_URL: "redis://redis"
+
+      S3_ENDPOINT: "http://minio:9000"
+      S3_PUBLIC_ENDPOINT: "http://127.0.0.1:9000"
+      S3_ACCESS_KEY: "minioadmin"
+      S3_SECRET_KEY: "minioadmin"
+      S3_BUCKET: "speckle-server"
+      S3_CREATE_BUCKET: "true"
+
+      FILE_SIZE_LIMIT_MB: 100
+      MAX_PROJECT_MODELS_PER_PAGE: 500
+
+      # TODO: Change this to a unique secret for this server
+      SESSION_SECRET: "TODO:ReplaceWithLongString"
+
+      STRATEGY_LOCAL: "true"
+
+      POSTGRES_URL: "postgres"
+      POSTGRES_USER: "speckle"
+      POSTGRES_PASSWORD: "speckle"
+      POSTGRES_DB: "speckle"
+      ENABLE_MP: "false"
+
+      LOG_PRETTY: "true"
+
+      FF_NEXT_GEN_FILE_IMPORTER_ENABLED: "true"
+      FF_LARGE_FILE_IMPORTS_ENABLED: "true"
+
+networks:
+  default:
+    name: speckle-server
+
+volumes:
+  postgres-data:
+  redis-data:
+  minio-data:

docker-compose.yml

@@ -12,7 +12,7 @@ services:
       POSTGRES_USER: speckle
       POSTGRES_PASSWORD: speckle
     volumes:
-      - postgres-data:/var/lib/postgresql/data/
+      - ./.volumes/postgres-data:/var/lib/postgresql/data/
     healthcheck:
       # the -U user has to match the POSTGRES_USER value
       test: ["CMD-SHELL", "pg_isready -U speckle"]
@@ -24,7 +24,7 @@ services:
     image: "valkey/valkey:8.1-alpine@sha256:0d27f0bca0249f61d060029a6aaf2e16b2c417d68d02a508e1dfb763fa2948b4"
     restart: always
     volumes:
-      - redis-data:/data
+      - ./.volumes/redis-data:/data
     healthcheck:
       test: ["CMD", "redis-cli", "--raw", "incr", "ping"]
       interval: 5s
@@ -36,7 +36,7 @@ services:
     command: server /data --console-address ":9001"
     restart: always
     volumes:
-      - minio-data:/data
+      - ./.volumes/minio-data:/data
     ports:
       - '127.0.0.1:9000:9000'
       - '127.0.0.1:9001:9001'
@@ -55,7 +55,7 @@ services:
     image: speckle/speckle-server:latest
     restart: always
     healthcheck:
-      test: 
+      test:
         - CMD
         - /nodejs/bin/node
         - -e
@@ -81,9 +81,9 @@ services:
 
       # TODO: Change thvolumes:
       REDIS_URL: "redis://redis"
-      
+
       S3_ENDPOINT: "http://minio:9000"
-      S3_PUBLIC_ENDPOINT: 'http://127.0.0.1:9000'
+      S3_PUBLIC_ENDPOINT: "http://127.0.0.1:9000"
       S3_ACCESS_KEY: "minioadmin"
       S3_SECRET_KEY: "minioadmin"
       S3_BUCKET: "speckle-server"
@@ -96,19 +96,17 @@ services:
       SESSION_SECRET: "TODO:ReplaceWithLongString"
 
       STRATEGY_LOCAL: "true"
-      DEBUG: "speckle:*"
 
       POSTGRES_URL: "postgres"
       POSTGRES_USER: "speckle"
       POSTGRES_PASSWORD: "speckle"
       POSTGRES_DB: "speckle"
       ENABLE_MP: "false"
-      
+
       LOG_PRETTY: "true"
-      
+
       FF_NEXT_GEN_FILE_IMPORTER_ENABLED: "true"
       FF_LARGE_FILE_IMPORTS_ENABLED: "true"
-      
 
 networks:
   default:

src/Speckle.Sdk/Api/Exceptions.cs

@@ -23,11 +23,15 @@ public class SpeckleGraphQLException : SpeckleException
 }
 
 /// <summary>
-/// Represents a "FORBIDDEN" or "UNAUTHORIZED" or "UNAUTHORIZED_ACCESS_ERROR" GraphQL error as an exception.
+/// Represents a "FORBIDDEN" or "UNAUTHENTICATED" or "UNAUTHORIZED" or "UNAUTHORIZED_ACCESS_ERROR" GraphQL error as an exception.
 /// https://www.apollographql.com/docs/apollo-server/v2/data/errors/#unauthenticated
 /// https://www.apollographql.com/docs/apollo-server/v2/data/errors/#forbidden
 /// https://github.com/specklesystems/speckle-server/blob/v2.23.18/packages/server/modules/shared/errors/index.ts#L34
 /// </summary>
+/// <remarks>
+/// Server is a bit inconsistent with these error codes, hence there's 4 different codes that mean "auth no work"
+/// Apollo no longer considers "FORBIDDEN" or "UNAUTHENTICATED" as built in error codes, so everything is custom anyway.
+/// </remarks>
 public sealed class SpeckleGraphQLForbiddenException : SpeckleGraphQLException
 {
   public SpeckleGraphQLForbiddenException() { }

src/Speckle.Sdk/Api/GraphQL/GraphQLErrorHandler.cs

@@ -28,9 +28,8 @@ internal static class GraphQLErrorHandler
       var ex = code switch
       {
         "GRAPHQL_PARSE_FAILED" or "GRAPHQL_VALIDATION_FAILED" => new SpeckleGraphQLInvalidQueryException(message),
-        "FORBIDDEN" or "UNAUTHENTICATED" or "UNAUTHORIZED_ACCESS_ERROR" => new SpeckleGraphQLForbiddenException(
+        "FORBIDDEN" or "UNAUTHENTICATED" or "UNAUTHORIZED" or "UNAUTHORIZED_ACCESS_ERROR" =>
-          message
+          new SpeckleGraphQLForbiddenException(message),
-        ),
         "STREAM_NOT_FOUND" => new SpeckleGraphQLStreamNotFoundException(message),
         "BAD_USER_INPUT" => new SpeckleGraphQLBadInputException(message),
         "INTERNAL_SERVER_ERROR" => new SpeckleGraphQLInternalErrorException(message),

src/Speckle.Sdk/Logging/SpecklePathProvider.cs

@@ -15,8 +15,8 @@ public static class SpecklePathProvider
 
   private const string ACCOUNTS_FOLDER_NAME = "Accounts";
 
-  private static string UserDataPathEnvVar => "SPECKLE_USERDATA_PATH";
+  public const string USER_DATA_PATH_ENV_VAR = "SPECKLE_USERDATA_PATH";
-  private static string? Path => Environment.GetEnvironmentVariable(UserDataPathEnvVar);
+  private static string? Path => Environment.GetEnvironmentVariable(USER_DATA_PATH_ENV_VAR);
 
   /// <summary>
   /// Get the installation path.

tests/Speckle.Sdk.Serialization.Testing/Speckle.Sdk.Serialization.Testing.csproj

@@ -4,6 +4,7 @@
     <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
+    <TestProjectAnalyserRules>true</TestProjectAnalyserRules>
   </PropertyGroup>
   <ItemGroup>
     <ProjectReference Include="..\..\src\Speckle.Sdk\Speckle.Sdk.csproj" />

tests/Speckle.Sdk.Testing/MoqTest.cs

@@ -8,7 +8,19 @@ public abstract class MoqTest : IDisposable
 {
   protected MoqTest() => Repository = new(MockBehavior.Strict);
 
-  public void Dispose() => Repository.VerifyAll();
+  protected virtual void Dispose(bool isDisposing)
+  {
+    if (isDisposing)
+    {
+      Repository.VerifyAll();
+    }
+  }
+
+  public void Dispose()
+  {
+    Dispose(true);
+    GC.SuppressFinalize(this);
+  }
 
   protected MockRepository Repository { get; private set; } = new(MockBehavior.Strict);
 

tests/Speckle.Sdk.Testing/Speckle.Sdk.Testing.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
   <PropertyGroup>
     <TargetFramework>net8.0</TargetFramework>
-    <IsTestProject>true</IsTestProject>
+    <TestProjectAnalyserRules>true</TestProjectAnalyserRules>
   </PropertyGroup>
   <ItemGroup>
     <PackageReference Include="Moq" />

tests/Speckle.Sdk.Tests.Integration/Fixtures.cs

@@ -15,6 +15,8 @@ using Speckle.Sdk.Tests.Integration.API.GraphQL.Resources;
 using Speckle.Sdk.Transports;
 using Version = Speckle.Sdk.Api.GraphQL.Models.Version;
 
+[assembly: AssemblyTrait("Category", "Integration")]
+
 namespace Speckle.Sdk.Tests.Integration;
 
 public static class Fixtures

tests/Speckle.Sdk.Tests.Performance/Speckle.Sdk.Tests.Performance.csproj

@@ -4,7 +4,7 @@
     <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>disable</Nullable>
-    <IsTestProject>true</IsTestProject>
+    <TestProjectAnalyserRules>true</TestProjectAnalyserRules>
   </PropertyGroup>
   <ItemGroup>
     <PackageReference Include="BenchmarkDotNet" />

tests/Speckle.Sdk.Tests.Unit/Credentials/AccountManagerTests.cs

@@ -9,7 +9,7 @@ using Speckle.Sdk.Testing;
 
 namespace Speckle.Sdk.Tests.Unit.Credentials;
 
-public class AccountManagerTests : MoqTest
+public sealed class AccountManagerTests : MoqTest
 {
   private class TestAccountFactory : IAccountFactory
   {
@@ -36,7 +36,9 @@ public class AccountManagerTests : MoqTest
   private readonly Mock<ISqLiteJsonCacheManager> _mockAccountStorage;
   private readonly Mock<ISqLiteJsonCacheManager> _mockAccountAddLockStorage;
 
+#pragma warning disable CA2213
   private readonly AccountManager _accountManager;
+#pragma warning restore CA2213
 
   public AccountManagerTests()
   {