huanld/speckle-sharp-sdk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Compare commits

base: huanld:3.15.3
Branches Tags
huanld:main huanld:jrm/stj-experiment huanld:jrm/avoid-deprecated-fields huanld:jrm/can-create-model-ingestion huanld:jrm/blobs-poc
huanld:3.16.0 huanld:3.17.0-net10-experiment huanld:3.17.0-stj-experiment huanld:3.15.3 huanld:3.15.2 huanld:3.15.2-alpha.1 huanld:3.15.1 huanld:3.15.0 huanld:3.14.0 huanld:3.14.0-alpha.9 huanld:3.14.0-alpha.8 huanld:3.14.0-alpha.7 huanld:3.14.0-alpha.6 huanld:3.14.0-alpha.5 huanld:3.14.0-alpha.4 huanld:3.14.0-alpha.3 huanld:3.14.0-alpha.2 huanld:3.14.0-alpha.1 huanld:3.13.1 huanld:3.13.0 huanld:3.12.0 huanld:3.11.1 huanld:3.11.0 huanld:3.10.1 huanld:3.10.0 huanld:3.10.0-alpha.3 huanld:3.10.0-alpha.2 huanld:3.11.0-alpha huanld:3.10.0-alpha huanld:3.9.0 huanld:3.8.0 huanld:3.7.1 huanld:3.7.0-alpha huanld:3.5.5 huanld:3.5.4 huanld:3.6.0-beta.1 huanld:3.5.3 huanld:3.5.2 huanld:3.5.2-alpha.1 huanld:3.5.1 huanld:3.5.1-rc.1 huanld:3.5.0 huanld:3.5.0-alpha.10 huanld:3.5.0-alpha.9 huanld:3.4.8 huanld:3.4.7 huanld:3.4.6 huanld:3.4.5 huanld:3.4.4 huanld:3.4.3 huanld:3.4.2 huanld:3.4.1 huanld:3.4.0 huanld:3.3.6 huanld:3.3.5 huanld:3.3.4 huanld:3.3.3 huanld:3.3.2 huanld:3.3.1 huanld:3.3.0 huanld:3.3.0-dev.1 huanld:3.2.3 huanld:3.2.2 huanld:3.2.1 huanld:3.2.0 huanld:3.1.8 huanld:3.1.7 huanld:3.1.6 huanld:3.1.5 huanld:3.1.4 huanld:3.1.3 huanld:3.1.2 huanld:3.1.1 huanld:3.1.0 huanld:3.1.0-dev.276 huanld:3.0.0
..
compare: huanld:3.5.2-alpha.1
Branches Tags
huanld:main huanld:jrm/stj-experiment huanld:jrm/avoid-deprecated-fields huanld:jrm/can-create-model-ingestion huanld:jrm/blobs-poc
huanld:3.16.0 huanld:3.17.0-net10-experiment huanld:3.17.0-stj-experiment huanld:3.15.3 huanld:3.15.2 huanld:3.15.2-alpha.1 huanld:3.15.1 huanld:3.15.0 huanld:3.14.0 huanld:3.14.0-alpha.9 huanld:3.14.0-alpha.8 huanld:3.14.0-alpha.7 huanld:3.14.0-alpha.6 huanld:3.14.0-alpha.5 huanld:3.14.0-alpha.4 huanld:3.14.0-alpha.3 huanld:3.14.0-alpha.2 huanld:3.14.0-alpha.1 huanld:3.13.1 huanld:3.13.0 huanld:3.12.0 huanld:3.11.1 huanld:3.11.0 huanld:3.10.1 huanld:3.10.0 huanld:3.10.0-alpha.3 huanld:3.10.0-alpha.2 huanld:3.11.0-alpha huanld:3.10.0-alpha huanld:3.9.0 huanld:3.8.0 huanld:3.7.1 huanld:3.7.0-alpha huanld:3.5.5 huanld:3.5.4 huanld:3.6.0-beta.1 huanld:3.5.3 huanld:3.5.2 huanld:3.5.2-alpha.1 huanld:3.5.1 huanld:3.5.1-rc.1 huanld:3.5.0 huanld:3.5.0-alpha.10 huanld:3.5.0-alpha.9 huanld:3.4.8 huanld:3.4.7 huanld:3.4.6 huanld:3.4.5 huanld:3.4.4 huanld:3.4.3 huanld:3.4.2 huanld:3.4.1 huanld:3.4.0 huanld:3.3.6 huanld:3.3.5 huanld:3.3.4 huanld:3.3.3 huanld:3.3.2 huanld:3.3.1 huanld:3.3.0 huanld:3.3.0-dev.1 huanld:3.2.3 huanld:3.2.2 huanld:3.2.1 huanld:3.2.0 huanld:3.1.8 huanld:3.1.7 huanld:3.1.6 huanld:3.1.5 huanld:3.1.4 huanld:3.1.3 huanld:3.1.2 huanld:3.1.1 huanld:3.1.0 huanld:3.1.0-dev.276 huanld:3.0.0

2 Commits
3.15.3 .. 3.5.2-alpha.1

Author SHA1 Message Date
Adam Hathcock 637997bd18 Report progress before saving SQLite
.NET Build and Publish / build (push) Has been cancelled
Details
2025-08-26 10:31:43 +01:00
Adam Hathcock 6c89748fd0 Report increment rather than total position 2025-08-26 10:04:14 +01:00
175 changed files with 1288 additions and 4521 deletions
Expand all files Collapse all files
.github/workflows/integration-test-callable-from-server-repo.yml
-61
View File
@@ -1,61 +0,0 @@
name: Integration Test
on:
workflow_call:
inputs:
speckle-sharp-sdk-ref:
required: true
type: string
jobs:
integration-test:
env:
CLIENT_DIR: "./client"
CLIENT_REPO: "specklesystems/speckle-sharp-sdk"
SERVER_DIR: "./server"
SERVER_REPO: "specklesystems/speckle-server-internal"
SOLUTION: "Speckle.Sdk.sln"
SPECKLE_SERVER_IMAGE: "speckle-server:local"
runs-on: ubuntu-latest
steps:
- name: Checkout ${{ env.CLIENT_REPO }}
uses: actions/checkout@v6
with:
path: ${{ env.CLIENT_DIR }}
repository: ${{ env.CLIENT_REPO }}
ref: ${{ inputs.speckle-sharp-sdk-ref }}
- name: Checkout ${{ env.SERVER_REPO }}
uses: actions/checkout@v6
with:
repository: ${{ env.SERVER_REPO }}
path: ${{ env.SERVER_DIR }}
- name: Setup .NET SDK
uses: actions/setup-dotnet@v5
with:
dotnet-version: 8.x.x
# cache: true
# cache-dependency-path: "**/packages.lock.json"
- name: 🏗️ Build Server
run: docker build --file "./packages/server/Dockerfile" --tag ${{ env.SPECKLE_SERVER_IMAGE }} .
working-directory: ${{ env.SERVER_DIR }}
- name: ⚙️ Spin up Server
run: docker compose --file "../${{ env.CLIENT_DIR }}/docker-compose-internal.yml" up --wait
working-directory: ${{ env.SERVER_DIR }}
env:
SPECKLE_SERVER_IMAGE: ${{ env.SPECKLE_SERVER_IMAGE }}
- name: 📦 Restore .NET Solution
run: dotnet restore ${{ env.SOLUTION }} --locked-mode
working-directory: ${{ env.CLIENT_DIR }}
- name: 🏗️ Build .NET Solution
run: dotnet build ${{ env.SOLUTION }} --configuration Release --no-restore -warnaserror
working-directory: ${{ env.CLIENT_DIR }}
- name: 🔨 Run .NET Integration Tests
run: dotnet test ${{ env.SOLUTION }} --filter "(Category=Integration)&(Server!=Public)" --configuration Release --no-build --no-restore --verbosity=normal
working-directory: ${{ env.CLIENT_DIR }}
.github/workflows/integration-test.yml
-62
View File
@@ -1,62 +0,0 @@
name: Integration Test
on:
workflow_call:
inputs:
docker-compose-file:
required: true
type: string
use-internal-image:
default: false
type: boolean
secrets:
CODECOV_TOKEN:
required: true
jobs:
integration-test:
env:
Solution: "Speckle.Sdk.sln"
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v6
- name: Setup .NET
uses: actions/setup-dotnet@v5
with:
dotnet-version: 8.x.x
cache: true
cache-dependency-path: "**/packages.lock.json"
- name: 🔐 Login to Github Container Registry
if: ${{ inputs.use-internal-image }}
uses: docker/login-action@v4
with:
registry: "ghcr.io"
username: ${{ github.actor }}
password: ${{ github.token }}
- name: ⚙️ Spin up Server
run: docker compose --file ${{ inputs.docker-compose-file }} up --wait
- name: 📦 Restore
run: dotnet restore ${{ env.Solution }} --locked-mode
- name: 🏗️ Build
run: dotnet build ${{ env.Solution }} --configuration Release --no-restore -warnaserror
- name: 🔨 Integration Tests against Public Server
if: ${{ !inputs.use-internal-image }}
run: dotnet test ${{ env.Solution }} --filter "(Category=Integration)&(Server!=Internal)" --configuration Release --no-build --no-restore --verbosity=normal /p:AltCover=true /p:AltCoverAttributeFilter=ExcludeFromCodeCoverage
- name: 🔨 Integration Tests against Internal Server
if: ${{ inputs.use-internal-image }}
run: dotnet test ${{ env.Solution }} --filter "(Category=Integration)&(Server!=Public)" --configuration Release --no-build --no-restore --verbosity=normal /p:AltCover=true /p:AltCoverAttributeFilter=ExcludeFromCodeCoverage
- name: Upload coverage reports to Codecov with GitHub Action
uses: codecov/codecov-action@v6
continue-on-error: true
with:
fail_ci_if_error: true
files: tests/**/coverage.xml
token: ${{ secrets.CODECOV_TOKEN }}
.github/workflows/pr.yml
+37 -56
View File
@@ -1,65 +1,46 @@
name: PR Test
name: .NET CI Build
on:
pull_request: {}
push:
branches:
- "main" # Need to run for codecov to compare against the BASE
pull_request:
jobs:
build:
env:
Solution: "Speckle.Sdk.sln"
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v6
- name: Checkout
uses: actions/checkout@v5
- name: Setup .NET
uses: actions/setup-dotnet@v5
with:
dotnet-version: 8.x.x
cache: true
cache-dependency-path: "**/packages.lock.json"
- name: 📦 Tool Restore
run: dotnet tool restore
- name: 📄 Format
run: dotnet csharpier check .
- name: 📦 Restore
run: dotnet restore ${{ env.Solution }} --locked-mode
- name: 🏗️ Build
run: dotnet build ${{ env.Solution }} --configuration Release --no-restore -warnaserror
- name: 🔨 Unit Tests
run: dotnet test ${{ env.Solution }} --configuration Release --filter "Category!=Integration" --no-build --no-restore --verbosity=normal /p:AltCover=true /p:AltCoverAttributeFilter=ExcludeFromCodeCoverage
- name: 🎁 Pack
run: dotnet pack ${{ env.Solution }} --configuration Release --no-build
- name: Upload coverage reports to Codecov with GitHub Action
uses: codecov/codecov-action@v6
continue-on-error: true
with:
fail_ci_if_error: true
files: tests/**/coverage.xml
token: ${{ secrets.CODECOV_TOKEN }}
integration-test-internal:
uses: "./.github/workflows/integration-test.yml"
with:
docker-compose-file: "docker-compose-internal.yml"
use-internal-image: true
secrets:
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
- name: Setup .NET
uses: actions/setup-dotnet@v4
with:
dotnet-version: 8.x.x
- uses: actions/cache@v4
with:
path: ~/.nuget/packages
key: ${{ runner.os }}-nuget-${{ hashFiles('**/packages.lock.json') }}
- id: set-version
name: Set version to output
run: |
SEMVER="3.0.99.${{ github.run_number }}"
FILE_VERSION=$(echo "$SEMVER" | sed -E 's/^([0-9]+\.[0-9]+\.[0-9]+).*/\1/')
FILE_VERSION="$FILE_VERSION.${{ github.run_number }}"
echo "semver=$SEMVER" >> "$GITHUB_OUTPUT"
echo "fileVersion=$FILE_VERSION" >> "$GITHUB_OUTPUT"
echo $SEMVER
echo $FILE_VERSION
- name: 🔫 Build All
run: ./build.sh
env:
SEMVER: ${{ steps.set-version.outputs.SEMVER }}
FILE_VERSION: ${{ steps.set-version.outputs.FILE_VERSION }}
integration-test-public:
uses: "./.github/workflows/integration-test.yml"
with:
docker-compose-file: "docker-compose.yml"
secrets:
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
- name: Upload coverage reports to Codecov with GitHub Action
uses: codecov/codecov-action@v5
with:
files: tests/**/coverage.xml
token: ${{ secrets.CODECOV_TOKEN }}
.github/workflows/release.yml
+16 -19
View File
@@ -2,27 +2,25 @@ name: .NET Build and Publish
on:
push:
tags: ["3.*.*"]
tags: ["3.*"]
jobs:
build:
runs-on: ubuntu-latest
environment:
name: "nuget.org"
permissions:
id-token: write # enable GitHub OIDC token issuance for this job
steps:
- name: Checkout
uses: actions/checkout@v6
uses: actions/checkout@v5
- name: Setup .NET
uses: actions/setup-dotnet@v5
uses: actions/setup-dotnet@v4
with:
dotnet-version: 8.x.x
cache: true
cache-dependency-path: "**/packages.lock.json"
- uses: actions/cache@v4
with:
path: ~/.nuget/packages
key: ${{ runner.os }}-nuget-${{ hashFiles('**/packages.lock.json') }}
- id: set-version
name: Set version to output
run: |
@@ -39,19 +37,18 @@ jobs:
echo $SEMVER
echo $FILE_VERSION
- name: 🔫 Build and Pack
run: ./build.sh pack
env:
SEMVER: ${{ steps.set-version.outputs.SEMVER }}
FILE_VERSION: ${{ steps.set-version.outputs.FILE_VERSION }}
- name: NuGet login (OIDC → temp API key)
uses: NuGet/login@v1
id: login
- name: Upload coverage reports to Codecov with GitHub Action
uses: codecov/codecov-action@v5
with:
user: ${{ secrets.NUGET_USER }}
files: tests/**/coverage.xml
token: ${{ secrets.CODECOV_TOKEN }}
- name: Push to nuget.org
run: dotnet nuget push output/*.nupkg --source "https://api.nuget.org/v3/index.json" --api-key ${{steps.login.outputs.NUGET_API_KEY}}
run: dotnet nuget push output/*.nupkg --source "https://api.nuget.org/v3/index.json" --api-key ${{secrets.CONNECTORS_NUGET_TOKEN }} --skip-duplicate
.tool-versions
-1
View File
@@ -1 +0,0 @@
dotnet 8.0.400
Directory.Build.targets
+2 -2
View File
@@ -1,8 +1,8 @@
<Project>
<PropertyGroup Condition="'$(IsTestProject)' == 'true' or '$(TestProjectAnalyserRules)' == 'true' ">
<PropertyGroup Condition="'$(IsTestProject)' == 'true'">
<NoWarn>
<!-- Things we need to test -->
CS0618;CA1034;CA2201;CA1051;CA1040;CA1724;CA1065;
CS0618;CA1034;CA2201;CA1051;CA1040;CA1724;
IDE0044;IDE0130;CA1508;
<!-- Analysers that provide no tangeable value to a test project -->
CA5394;CA2007;CA1852;CA1819;CA1711;CA1063;CA1816;CA2234;CS8618;CA1054;CA1810;CA2208;CA1019;CA1831;
Directory.Packages.props
+1 -1
View File
@@ -15,8 +15,8 @@
<PackageVersion Include="Microsoft.NET.Test.Sdk" Version="17.13.0" />
<PackageVersion Include="Microsoft.Extensions.DependencyInjection" Version="[2.2.0,)" />
<PackageVersion Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="[2.2.0,)" />
<PackageVersion Include="Microsoft.Bcl.AsyncInterfaces" Version="[9.0.4,)" />
<PackageVersion Include="Microsoft.Extensions.Logging" Version="[2.2.0,)" />
<PackageVersion Include="Microsoft.Bcl.AsyncInterfaces" Version="[5.0.0,)" />
<PackageVersion Include="Moq" Version="4.20.72" />
<PackageVersion Include="Newtonsoft.Json.Schema" Version="4.0.1" />
<PackageVersion Include="Open.ChannelExtensions" Version="9.1.0" />
README.md
+2 -4
View File
@@ -65,12 +65,10 @@ Docs are a bit patchy [https://docs.speckle.systems/developers/looking-for-devel
### Tests
There are several test projects. It is a requirement that all tests pass for PRs to be merged.
The Integration test projects require a local server to be running.
You must have docker installed. Then you can run `docker compose up` from the root of the repo to start the required containers.
In CI, they will be run against both the public and private versions of the server.
It is important that we remain compatible with both server versions.
You must have docker installed. Then you can run `docker compose up --wait` from the root of the repo to start the required containers.
## Contributing
Before embarking on submitting a patch, please make sure you read:
Speckle.Sdk.sln
-2
View File
@@ -27,7 +27,6 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "config", "config", "{DA2AED
CodeMetricsConfig.txt = CodeMetricsConfig.txt
Directory.Build.Targets = Directory.Build.Targets
.config\dotnet-tools.json = .config\dotnet-tools.json
docker-compose-internal.yml = docker-compose-internal.yml
EndProjectSection
EndProject
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "build", "build", "{58D37DA9-F948-48CA-9A73-F5BBBD533DBF}"
@@ -42,7 +41,6 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "workflows", "workflows", "{
ProjectSection(SolutionItems) = preProject
.github\workflows\pr.yml = .github\workflows\pr.yml
.github\workflows\release.yml = .github\workflows\release.yml
.github\workflows\integration-test.yml = .github\workflows\integration-test.yml
EndProjectSection
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Speckle.Sdk.Tests.Performance", "tests\Speckle.Sdk.Tests.Performance\Speckle.Sdk.Tests.Performance.csproj", "{870E3396-E6F7-43AE-B120-E651FA4F46BD}"
Speckle.Sdk.slnx
-3
View File
@@ -10,7 +10,6 @@
<File Path="Directory.Build.props" />
<File Path="Directory.Build.Targets" />
<File Path="Directory.Packages.props" />
<File Path="docker-compose-internal.yml" />
<File Path="docker-compose.yml" />
<File Path="global.json" />
<File Path="README.md" />
@@ -18,8 +17,6 @@
<File Path=".github\git-commit-instructions.md" />
</Folder>
<Folder Name="/config/workflows/">
<File Path=".github/workflows/integration-test-callable-from-server-repo.yml" />
<File Path=".github/workflows/integration-test.yml" />
<File Path=".github/workflows/pr.yml" />
<File Path=".github/workflows/release.yml" />
</Folder>
docker-compose-internal.yml
-115
View File
@@ -1,115 +0,0 @@
name: "speckle-server"
services:
####
# Speckle Server dependencies
#######
postgres:
image: "postgres:16.4-alpine3.20@sha256:d898b0b78a2627cb4ee63464a14efc9d296884f1b28c841b0ab7d7c42f1fffdf"
restart: always
environment:
POSTGRES_DB: speckle
POSTGRES_USER: speckle
POSTGRES_PASSWORD: speckle
volumes:
- ./.volumes/postgres-data:/var/lib/postgresql/data/
healthcheck:
# the -U user has to match the POSTGRES_USER value
test: ["CMD-SHELL", "pg_isready -U speckle"]
interval: 5s
timeout: 5s
retries: 30
redis:
image: "valkey/valkey:8.1-alpine@sha256:0d27f0bca0249f61d060029a6aaf2e16b2c417d68d02a508e1dfb763fa2948b4"
restart: always
volumes:
- ./.volumes/redis-data:/data
healthcheck:
test: ["CMD", "redis-cli", "--raw", "incr", "ping"]
interval: 5s
timeout: 5s
retries: 30
minio:
image: "minio/minio:RELEASE.2023-10-25T06-33-25Z"
command: server /data --console-address ":9001"
restart: always
volumes:
- ./.volumes/minio-data:/data
ports:
- '127.0.0.1:9000:9000'
- '127.0.0.1:9001:9001'
healthcheck:
test:
[
"CMD-SHELL",
"curl -s -o /dev/null http://127.0.0.1:9000/minio/index.html",
]
interval: 5s
timeout: 30s
retries: 30
start_period: 10s
speckle-server:
image: ${SPECKLE_SERVER_IMAGE:-ghcr.io/specklesystems/speckle-server:latest}
restart: always
healthcheck:
test:
- CMD
- /nodejs/bin/node
- -e
- "try { require('node:http').request({headers: {'Content-Type': 'application/json'}, port:3000, hostname:'127.0.0.1', path:'/readiness', method: 'GET', timeout: 2000 }, (res) => { body = ''; res.on('data', (chunk) => {body += chunk;}); res.on('end', () => {process.exit(Number(res.statusCode != 200 || body.toLowerCase().includes('error')));}); }).end(); } catch { process.exit(1); }"
interval: 10s
timeout: 10s
retries: 3
start_period: 90s
ports:
- "0.0.0.0:3000:3000"
depends_on:
postgres:
condition: service_healthy
redis:
condition: service_healthy
minio:
condition: service_healthy
environment:
# TODO: Change this to the URL of the speckle server, as accessed from the network
CANONICAL_URL: "http://127.0.0.1:8080"
SPECKLE_AUTOMATE_URL: "http://127.0.0.1:3030"
FRONTEND_ORIGIN: "http://127.0.0.1:8081"
# TODO: Change thvolumes:
REDIS_URL: "redis://redis"
S3_ENDPOINT: "http://minio:9000"
S3_PUBLIC_ENDPOINT: "http://127.0.0.1:9000"
S3_ACCESS_KEY: "minioadmin"
S3_SECRET_KEY: "minioadmin"
S3_BUCKET: "speckle-server"
S3_CREATE_BUCKET: "true"
FILE_SIZE_LIMIT_MB: 100
MAX_PROJECT_MODELS_PER_PAGE: 500
# TODO: Change this to a unique secret for this server
SESSION_SECRET: "TODO:ReplaceWithLongString"
STRATEGY_LOCAL: "true"
POSTGRES_URL: 'postgres://speckle:speckle@postgres:5432/speckle'
ENABLE_MP: "false"
LOG_PRETTY: "true"
FF_NEXT_GEN_FILE_IMPORTER_ENABLED: "true"
FF_LARGE_FILE_IMPORTS_ENABLED: "true"
networks:
default:
name: speckle-server
volumes:
postgres-data:
redis-data:
minio-data:
docker-compose.yml
+10 -8
View File
@@ -12,7 +12,7 @@ services:
POSTGRES_USER: speckle
POSTGRES_PASSWORD: speckle
volumes:
- ./.volumes/postgres-data:/var/lib/postgresql/data/
- postgres-data:/var/lib/postgresql/data/
healthcheck:
# the -U user has to match the POSTGRES_USER value
test: ["CMD-SHELL", "pg_isready -U speckle"]
@@ -24,7 +24,7 @@ services:
image: "valkey/valkey:8.1-alpine@sha256:0d27f0bca0249f61d060029a6aaf2e16b2c417d68d02a508e1dfb763fa2948b4"
restart: always
volumes:
- ./.volumes/redis-data:/data
- redis-data:/data
healthcheck:
test: ["CMD", "redis-cli", "--raw", "incr", "ping"]
interval: 5s
@@ -36,7 +36,7 @@ services:
command: server /data --console-address ":9001"
restart: always
volumes:
- ./.volumes/minio-data:/data
- minio-data:/data
ports:
- '127.0.0.1:9000:9000'
- '127.0.0.1:9001:9001'
@@ -55,7 +55,7 @@ services:
image: speckle/speckle-server:latest
restart: always
healthcheck:
test:
test:
- CMD
- /nodejs/bin/node
- -e
@@ -81,9 +81,9 @@ services:
# TODO: Change thvolumes:
REDIS_URL: "redis://redis"
S3_ENDPOINT: "http://minio:9000"
S3_PUBLIC_ENDPOINT: "http://127.0.0.1:9000"
S3_PUBLIC_ENDPOINT: 'http://127.0.0.1:9000'
S3_ACCESS_KEY: "minioadmin"
S3_SECRET_KEY: "minioadmin"
S3_BUCKET: "speckle-server"
@@ -96,17 +96,19 @@ services:
SESSION_SECRET: "TODO:ReplaceWithLongString"
STRATEGY_LOCAL: "true"
DEBUG: "speckle:*"
POSTGRES_URL: "postgres"
POSTGRES_USER: "speckle"
POSTGRES_PASSWORD: "speckle"
POSTGRES_DB: "speckle"
ENABLE_MP: "false"
LOG_PRETTY: "true"
FF_NEXT_GEN_FILE_IMPORTER_ENABLED: "true"
FF_LARGE_FILE_IMPORTS_ENABLED: "true"
networks:
default:
src/Speckle.Automate.Sdk/AutomationContext.cs
+1 -5
View File
@@ -63,7 +63,7 @@ internal sealed class AutomationContext(IOperations operations) : IAutomationCon
);
}
Base rootObject = await operations
Base? rootObject = await operations
.Receive2(
SpeckleClient.ServerUrl,
AutomationRunData.ProjectId,
@@ -74,10 +74,6 @@ internal sealed class AutomationContext(IOperations operations) : IAutomationCon
)
.ConfigureAwait(false);
await SpeckleClient
.Version.Received(new(version.id, AutomationRunData.ProjectId, "automate_function"), cancellationToken)
.ConfigureAwait(false);
Console.WriteLine($"It took {Elapsed.TotalSeconds} seconds to receive the speckle version {versionId}");
return rootObject;
}
src/Speckle.Automate.Sdk/packages.lock.json
+5 -7
View File
@@ -281,6 +281,7 @@
"type": "Project",
"dependencies": {
"GraphQL.Client": "[6.0.0, )",
"Microsoft.Bcl.AsyncInterfaces": "[5.0.0, )",
"Microsoft.CSharp": "[4.7.0, )",
"Microsoft.Data.Sqlite": "[7.0.5, )",
"Microsoft.Extensions.DependencyInjection.Abstractions": "[2.2.0, )",
@@ -291,10 +292,7 @@
}
},
"speckle.sdk.dependencies": {
"type": "Project",
"dependencies": {
"Microsoft.Bcl.AsyncInterfaces": "[9.0.4, )"
}
"type": "Project"
},
"GraphQL.Client": {
"type": "CentralTransitive",
@@ -309,9 +307,9 @@
},
"Microsoft.Bcl.AsyncInterfaces": {
"type": "CentralTransitive",
"requested": "[9.0.4, )",
"resolved": "9.0.4",
"contentHash": "9VGI5kxIvrNG2mqLQZnUR6y/3fcnygD8eNpHR+CqfbnIXvea6nehnYknDKQTxZVPMpzpNca+7DxLBmpdB3q0Bw==",
"requested": "[5.0.0, )",
"resolved": "8.0.0",
"contentHash": "3WA9q9yVqJp222P3x1wYIGDAkpjAku0TMUaaQV22g6L67AI0LdOIrVS7Ht2vJfLHGSPVuqN94vIr15qn+HEkHw==",
"dependencies": {
"System.Threading.Tasks.Extensions": "4.5.4"
}
src/Speckle.Objects/Geometry/SolidX.cs
-6
View File
@@ -1,6 +0,0 @@
using Speckle.Sdk.Models;
namespace Speckle.Objects.Geometry;
[SpeckleType("Objects.Geometry.SolidX")]
public class SolidX : RawEncodedObject;
src/Speckle.Objects/Interfaces.cs
+9 -2
View File
@@ -2,7 +2,6 @@ using Speckle.Objects.Geometry;
using Speckle.Objects.Other;
using Speckle.Objects.Primitive;
using Speckle.Sdk.Models;
using Speckle.Sdk.Models.Data;
namespace Speckle.Objects;
@@ -111,7 +110,15 @@ public interface IDisplayValue<out T> : ISpeckleObject
#region Data objects
public interface IDataObject : IProperties, IDisplayValue<IReadOnlyList<Base>>, ISpeckleObject
/// <summary>
/// Specifies properties on objects to be used for data-based workflows
/// </summary>
public interface IProperties : ISpeckleObject
{
Dictionary<string, object?> properties { get; }
}
public interface IDataObject : IProperties, IDisplayValue<IReadOnlyList<Base>>
{
/// <summary>
/// The name of the object, primarily used to decorate the object for consumption in frontend and other apps
src/Speckle.Objects/Other/Camera.cs
-32
View File
@@ -1,32 +0,0 @@
using Speckle.Objects.Geometry;
using Speckle.Sdk.Models;
namespace Speckle.Objects.Other;
/// <summary>
/// Camera class to represent a perspective camera for a 3D view.
/// </summary>
/// <remarks>Assumes a Z-up, right-handed convention for orientation vectors</remarks>
[SpeckleType("Objects.Other.Camera")]
public class Camera : Base
{
/// <summary>
/// The name of the view that is created by this camera
/// </summary>
public required string name { get; set; }
/// <summary>
/// The location of the camera
/// </summary>
public required Point position { get; set; }
/// <summary>
/// The unit up vector of the camera
/// </summary>
public required Vector up { get; set; }
/// <summary>
/// The unit forward vector of the camera
/// </summary>
public required Vector forward { get; set; }
}
src/Speckle.Objects/Other/RawEncoding.cs
-2
View File
@@ -20,6 +20,4 @@ public class RawEncoding : Base // note: at this stage, since we're using this f
public static class RawEncodingFormats
{
public const string RHINO_3DM = "3dm";
public const string ACAD_DWG = "dwg";
public const string ACAD_SAT = "sat";
}
src/Speckle.Objects/Other/RenderMaterial.cs
+1 -1
View File
@@ -35,6 +35,6 @@ public class RenderMaterial : Base
public Color emissiveColor
{
get => Color.FromArgb(emissive);
set => emissive = value.ToArgb();
set => diffuse = value.ToArgb();
}
}
src/Speckle.Objects/packages.lock.json
+5 -7
View File
@@ -228,6 +228,7 @@
"type": "Project",
"dependencies": {
"GraphQL.Client": "[6.0.0, )",
"Microsoft.Bcl.AsyncInterfaces": "[5.0.0, )",
"Microsoft.CSharp": "[4.7.0, )",
"Microsoft.Data.Sqlite": "[7.0.5, )",
"Microsoft.Extensions.DependencyInjection.Abstractions": "[2.2.0, )",
@@ -238,10 +239,7 @@
}
},
"speckle.sdk.dependencies": {
"type": "Project",
"dependencies": {
"Microsoft.Bcl.AsyncInterfaces": "[9.0.4, )"
}
"type": "Project"
},
"GraphQL.Client": {
"type": "CentralTransitive",
@@ -256,9 +254,9 @@
},
"Microsoft.Bcl.AsyncInterfaces": {
"type": "CentralTransitive",
"requested": "[9.0.4, )",
"resolved": "9.0.4",
"contentHash": "9VGI5kxIvrNG2mqLQZnUR6y/3fcnygD8eNpHR+CqfbnIXvea6nehnYknDKQTxZVPMpzpNca+7DxLBmpdB3q0Bw==",
"requested": "[5.0.0, )",
"resolved": "5.0.0",
"contentHash": "W8DPQjkMScOMTtJbPwmPyj9c3zYSFGawDW3jwlBOOsnY+EzZFLgNQ/UMkK35JmkNOVPdCyPr2Tw7Vv9N+KA3ZQ==",
"dependencies": {
"System.Threading.Tasks.Extensions": "4.5.4"
}
src/Speckle.Sdk.Dependencies/ISdkActivity.cs
-1
View File
@@ -5,7 +5,6 @@ public interface ISdkActivity : IDisposable
void SetTag(string key, object? value);
void RecordException(Exception e);
string TraceId { get; }
string SpanId { get; }
void SetStatus(SdkActivityStatusCode code);
void InjectHeaders(Action<string, string> header);
src/Speckle.Sdk.Dependencies/ISdkActivityFactory.cs
+1 -13
View File
@@ -1,20 +1,8 @@
using System.Runtime.CompilerServices;
using Speckle.Connectors.Logging;
namespace Speckle.Sdk.Logging;
public interface ISdkActivityFactory : IDisposable
{
ISdkActivity? Start(
string? name = null,
SdkActivityKind kind = SdkActivityKind.Internal,
[CallerMemberName] string source = ""
);
ISdkActivity? StartRemote(
string traceContext,
SdkActivityKind kind,
string? name = null,
[CallerMemberName] string source = ""
);
ISdkActivity? Start(string? name = default, [CallerMemberName] string source = "");
}
src/Speckle.Sdk.Dependencies/RepackedChannel.cs
-49
View File
@@ -1,49 +0,0 @@
using System.Threading.Channels;
namespace Speckle.Sdk.Dependencies;
/// <summary>
/// For various reasons related to our use of ILRepack.FullAuto,
/// we cannot use Channels from the SDK project.
/// We have to keep usage of it inside the Sdk.Dependencies project.
///
/// For the sake of quick development, I've wrapped the <see cref="Channel"/> class here in a type
/// that is safe to use from the SDK project.
///
/// As and when we need more functions, we can add them here.
///
/// And yes... I'm not very happy about the way we've set this up
/// </summary>
/// <typeparam name="T"></typeparam>
public sealed class RepackedChannel<T>
{
private readonly Channel<T> _channel;
public RepackedChannel(int capacity, bool singleReader, bool singleWriter)
{
_channel = Channel.CreateBounded<T>(
new BoundedChannelOptions(capacity)
{
FullMode = BoundedChannelFullMode.Wait,
SingleReader = singleReader,
SingleWriter = singleWriter,
}
);
}
public void CompleteWriter() => _channel.Writer.Complete();
public ValueTask WriteAsync(T item, CancellationToken cancellationToken) =>
_channel.Writer.WriteAsync(item, cancellationToken);
public IAsyncEnumerable<T> ReadAllAsync(CancellationToken cancellationToken) =>
_channel.Reader.ReadAllAsync(cancellationToken);
// public async Task ReadAllAsync(Func<T, Task> callback, CancellationToken cancellationToken)
// {
// await foreach (T item in _channel.Reader.ReadAllAsync(cancellationToken))
// {
// await callback.Invoke(item).ConfigureAwait(false);
// }
// }
}
src/Speckle.Sdk.Dependencies/Speckle.Sdk.Dependencies.csproj
-32
View File
@@ -28,36 +28,4 @@
<PackageReference Include="Open.ChannelExtensions" PrivateAssets="all" />
<PackageReference Include="System.Threading.Channels" PrivateAssets="all" />
</ItemGroup>
<ItemGroup Condition=" '$(TargetFramework)' == 'netstandard2.0'">
<PackageReference Include="Microsoft.Bcl.AsyncInterfaces" />
</ItemGroup>
<Target Name="BeforeILRepackPrepareBuild" BeforeTargets="ILRepackPrepareBuild">
<ItemGroup>
<!--
We're Being selective about which assemblies we're il-repacking
Avoiding repacling `Microsoft.Bcl.AsyncInterfaces.dll` because we need types like `ValueTask` and `IAsyncEnumerable` to be external
Yes, this does beg the question, why are we using `IlRepack.FullAuto` instead of raw ILRepack. Well the truth is, I'd like to move away from FullAuto
since it's unmaintaned and is lagging behind ILRepack version.
-->
<_ILRepackIncludeAssemblies_Items Include="$(OutputPath)System.Numerics.Vectors.dll" />
<_ILRepackIncludeAssemblies_Items Include="$(OutputPath)System.Runtime.CompilerServices.Unsafe.dll" />
<_ILRepackIncludeAssemblies_Items Include="$(OutputPath)System.Memory.dll" />
<_ILRepackIncludeAssemblies_Items Include="$(OutputPath)Open.ChannelExtensions.dll" />
<_ILRepackIncludeAssemblies_Items Include="$(OutputPath)System.Threading.Channels.dll" />
<_ILRepackIncludeAssemblies_Items Include="$(OutputPath)System.Collections.Immutable.dll" />
<_ILRepackIncludeAssemblies_Items Include="$(OutputPath)Polly.dll" />
<_ILRepackIncludeAssemblies_Items Include="$(OutputPath)Polly.Contrib.WaitAndRetry.dll" />
<_ILRepackIncludeAssemblies_Items Include="$(OutputPath)Polly.Extensions.Http.dll" />
<_ILRepackIncludeAssemblies_Items Include="$(OutputPath)Microsoft.Extensions.ObjectPool.dll" />
<_ILRepackExcludeAssemblies_Items Include="$(OutputPath)*.dll" Exclude="@(_ILRepackIncludeAssemblies_Items)" />
</ItemGroup>
<Message
Text="These are the packages we are NOT ilrepacking '@(_ILRepackExcludeAssemblies_Items)'"
Importance="high"
/>
<PropertyGroup>
<ILRepackExcludeAssemblies>@(_ILRepackExcludeAssemblies_Items)</ILRepackExcludeAssemblies>
</PropertyGroup>
</Target>
</Project>
src/Speckle.Sdk.Dependencies/SpeckleActivityKind.cs
-30
View File
@@ -1,30 +0,0 @@
namespace Speckle.Connectors.Logging;
public enum SdkActivityKind
{
/// <summary>
/// Default value.
/// Indicates that the Activity represents an internal operation within an application, as opposed to an operations with remote parents or children.
/// </summary>
Internal = 0,
/// <summary>
/// Server activity represents request incoming from external component.
/// </summary>
Server = 1,
/// <summary>
/// Client activity represents outgoing request to the external component.
/// </summary>
Client = 2,
/// <summary>
/// Producer activity represents output provided to external components.
/// </summary>
Producer = 3,
/// <summary>
/// Consumer activity represents output received from an external component.
/// </summary>
Consumer = 4,
}
src/Speckle.Sdk.Dependencies/packages.lock.json
+9 -9
View File
@@ -11,15 +11,6 @@
"ILRepack": "2.0.33"
}
},
"Microsoft.Bcl.AsyncInterfaces": {
"type": "Direct",
"requested": "[9.0.4, )",
"resolved": "9.0.4",
"contentHash": "9VGI5kxIvrNG2mqLQZnUR6y/3fcnygD8eNpHR+CqfbnIXvea6nehnYknDKQTxZVPMpzpNca+7DxLBmpdB3q0Bw==",
"dependencies": {
"System.Threading.Tasks.Extensions": "4.5.4"
}
},
"Microsoft.Extensions.ObjectPool": {
"type": "Direct",
"requested": "[9.0.4, )",
@@ -160,6 +151,15 @@
"dependencies": {
"System.Runtime.CompilerServices.Unsafe": "4.5.3"
}
},
"Microsoft.Bcl.AsyncInterfaces": {
"type": "CentralTransitive",
"requested": "[5.0.0, )",
"resolved": "9.0.4",
"contentHash": "9VGI5kxIvrNG2mqLQZnUR6y/3fcnygD8eNpHR+CqfbnIXvea6nehnYknDKQTxZVPMpzpNca+7DxLBmpdB3q0Bw==",
"dependencies": {
"System.Threading.Tasks.Extensions": "4.5.4"
}
}
},
"net8.0": {
src/Speckle.Sdk/Api/Blob/BlobApi.cs
+24 -1
View File
@@ -193,7 +193,30 @@ public sealed class BlobApi : IBlobApi
using var response = await _unauthedClient.SendAsync(requestMessage, cancellationToken).ConfigureAwait(false);
response.EnsureSuccessStatusCode();
return BlobApiHelpers.ParseEtagHeader(response.Headers);
return ParseEtagHeader(response.Headers);
}
private static string ParseEtagHeader(HttpResponseHeaders headers)
{
if (!headers.TryGetValues("ETag", out var etagValues))
{
throw new ArgumentException(
"Response does not have an ETag attached to it, cannot use this as an upload",
nameof(headers)
);
}
var etagValuesArray = etagValues.ToArray();
if (etagValuesArray.Length != 1)
{
throw new ArgumentException(
$"Expected Etag header to have a single value but got {etagValuesArray.Length}",
nameof(headers)
);
}
return etagValuesArray[0];
}
/// <summary>
src/Speckle.Sdk/Api/Exceptions.cs
+1 -6
View File
@@ -23,15 +23,10 @@ public class SpeckleGraphQLException : SpeckleException
}
/// <summary>
/// Represents a "FORBIDDEN" or "UNAUTHENTICATED" or "UNAUTHORIZED" or "UNAUTHORIZED_ACCESS_ERROR" GraphQL error as an exception.
/// Represents a "FORBIDDEN" or "UNAUTHORIZED" GraphQL error as an exception.
/// https://www.apollographql.com/docs/apollo-server/v2/data/errors/#unauthenticated
/// https://www.apollographql.com/docs/apollo-server/v2/data/errors/#forbidden
/// https://github.com/specklesystems/speckle-server/blob/v2.23.18/packages/server/modules/shared/errors/index.ts#L34
/// </summary>
/// <remarks>
/// Server is a bit inconsistent with these error codes, hence there's 4 different codes that mean "auth no work"
/// Apollo no longer considers "FORBIDDEN" or "UNAUTHENTICATED" as built in error codes, so everything is custom anyway.
/// </remarks>
public sealed class SpeckleGraphQLForbiddenException : SpeckleGraphQLException
{
public SpeckleGraphQLForbiddenException() { }
src/Speckle.Sdk/Api/GraphQL/Client.cs
+2 -24
View File
@@ -35,7 +35,6 @@ public sealed class Client : ISpeckleGraphQLClient, IClient
public WorkspaceResource Workspace { get; }
public ServerResource Server { get; }
public FileImportResource FileImport { get; }
public ModelIngestionResource Ingestion { get; }
public Uri ServerUrl => new(Account.serverInfo.url);
@@ -72,7 +71,6 @@ public sealed class Client : ISpeckleGraphQLClient, IClient
Workspace = new(this);
Server = new(this);
FileImport = new(this, blobApiFactory.Create(account));
Ingestion = new(this);
}
[AutoInterfaceIgnore]
@@ -87,26 +85,6 @@ public sealed class Client : ISpeckleGraphQLClient, IClient
catch (Exception ex) when (!ex.IsFatal()) { }
}
/// <summary>
/// Ensure the <see cref="GQLClient"/>'s websocket is fully initialized.
/// <br/>
/// You don't <i>need</i> to call this function, if you don't, then it will be setup for you when you call <see cref="SubscribeTo"/> (e.g. when you create a <see cref="Subscription"/>),
/// but due to <see cref="GraphQL"/>'s WebSocket implementation, it's not awaited (deferred) thus the subscription make take a while to actually be setup.
/// </summary>
/// <remarks>
/// We only use websockets for GraphQL subscriptions, so if you're not using subscriptions, don't call this
///
/// Note. due to other sources (potentially on the GraphQL side) you still need a ~100ms delay between setting up the subscription, and being able to relaibly trigger it
/// This should only really negatively affect test projects.
/// </remarks>
public async Task InitializeWebsocket()
{
if (GQLClient.WebSocketSubProtocol is null)
{
await GQLClient.InitializeWebsocketConnection().ConfigureAwait(false);
}
}
internal async Task<T> ExecuteWithResiliencePolicies<T>(Func<Task<T>> func) =>
await GraphQLRetry
.ExecuteAsync<T, SpeckleGraphQLInternalErrorException>(
@@ -154,10 +132,10 @@ public sealed class Client : ISpeckleGraphQLClient, IClient
activity?.SetStatus(SdkActivityStatusCode.Ok);
return ret;
}
catch (Exception ex)
catch (Exception)
{
activity?.SetStatus(SdkActivityStatusCode.Error);
activity?.RecordException(ex);
// Don't record exception as it's rethrown.
throw;
}
}
src/Speckle.Sdk/Api/GraphQL/Enums/FileUploadConversionStatus.cs
+2 -5
View File
@@ -1,9 +1,6 @@
// ReSharper disable InconsistentNaming
namespace Speckle.Sdk.Api.GraphQL.Enums;
namespace Speckle.Sdk.Api.GraphQL.Enums;
/// <remarks>
/// This enum isn't explicitly defined in the schema, instead its usages are int typed (But represent an enum)
/// </remarks>
//This enum isn't explicitly defined in the schema, instead its usages are int typed (But represent an enum)
public enum FileUploadConversionStatus
{
Queued = 0,
src/Speckle.Sdk/Api/GraphQL/Enums/ModelIngestionStatus.cs
-14
View File
@@ -1,14 +0,0 @@
// ReSharper disable InconsistentNaming
namespace Speckle.Sdk.Api.GraphQL.Enums;
/// <remarks>
/// string based enum
/// </remarks>
public enum ModelIngestionStatus
{
cancelled,
failed,
processing,
queued,
success,
}
src/Speckle.Sdk/Api/GraphQL/Enums/ProjectCommentsUpdatedMessageType.cs
-3
View File
@@ -1,8 +1,5 @@
namespace Speckle.Sdk.Api.GraphQL.Enums;
/// <remarks>
/// string based enum
/// </remarks>
public enum ProjectCommentsUpdatedMessageType
{
ARCHIVED,
src/Speckle.Sdk/Api/GraphQL/Enums/ProjectFileImportUpdatedMessageType.cs
-3
View File
@@ -1,8 +1,5 @@
namespace Speckle.Sdk.Api.GraphQL.Enums;
/// <remarks>
/// string based enum
/// </remarks>
public enum ProjectFileImportUpdatedMessageType
{
CREATED,
src/Speckle.Sdk/Api/GraphQL/Enums/ProjectModelIngestionUpdatedMessageType.cs
-13
View File
@@ -1,13 +0,0 @@
// ReSharper disable InconsistentNaming
namespace Speckle.Sdk.Api.GraphQL.Enums;
/// <remarks>
/// string based enum
/// </remarks>
public enum ProjectModelIngestionUpdatedMessageType
{
cancellationRequested,
created,
deleted,
updated,
}
src/Speckle.Sdk/Api/GraphQL/Enums/ProjectModelsUpdatedMessageType.cs
-3
View File
@@ -1,8 +1,5 @@
namespace Speckle.Sdk.Api.GraphQL.Enums;
/// <remarks>
/// string based enum
/// </remarks>
public enum ProjectModelsUpdatedMessageType
{
CREATED,
src/Speckle.Sdk/Api/GraphQL/Enums/ProjectPendingModelsUpdatedMessageType.cs
-3
View File
@@ -1,8 +1,5 @@
namespace Speckle.Sdk.Api.GraphQL.Enums;
/// <remarks>
/// string based enum
/// </remarks>
public enum ProjectPendingModelsUpdatedMessageType
{
CREATED,
src/Speckle.Sdk/Api/GraphQL/Enums/ProjectUpdatedMessageType.cs
-3
View File
@@ -1,8 +1,5 @@
namespace Speckle.Sdk.Api.GraphQL.Enums;
/// <remarks>
/// string based enum
/// </remarks>
public enum ProjectUpdatedMessageType
{
DELETED,
src/Speckle.Sdk/Api/GraphQL/Enums/ProjectVersionsUpdatedMessageType.cs
-3
View File
@@ -1,8 +1,5 @@
namespace Speckle.Sdk.Api.GraphQL.Enums;
/// <remarks>
/// string based enum
/// </remarks>
public enum ProjectVersionsUpdatedMessageType
{
CREATED,
src/Speckle.Sdk/Api/GraphQL/Enums/ProjectVisibility.cs
-4
View File
@@ -1,9 +1,5 @@
// ReSharper disable InconsistentNaming
namespace Speckle.Sdk.Api.GraphQL.Enums;
/// <remarks>
/// string based enum
/// </remarks>
public enum ProjectVisibility
{
Private,
src/Speckle.Sdk/Api/GraphQL/Enums/ResourceType.cs
+1 -5
View File
@@ -1,9 +1,5 @@
// ReSharper disable InconsistentNaming
namespace Speckle.Sdk.Api.GraphQL.Enums;
namespace Speckle.Sdk.Api.GraphQL.Enums;
/// <remarks>
/// string based enum
/// </remarks>
public enum ResourceType
{
commit,
src/Speckle.Sdk/Api/GraphQL/Enums/UserProjectsUpdatedMessageType.cs
-3
View File
@@ -1,8 +1,5 @@
namespace Speckle.Sdk.Api.GraphQL.Enums;
/// <remarks>
/// string based enum
/// </remarks>
public enum UserProjectsUpdatedMessageType
{
ADDED,
src/Speckle.Sdk/Api/GraphQL/GraphQLErrorHandler.cs
+1 -2
View File
@@ -28,8 +28,7 @@ internal static class GraphQLErrorHandler
var ex = code switch
{
"GRAPHQL_PARSE_FAILED" or "GRAPHQL_VALIDATION_FAILED" => new SpeckleGraphQLInvalidQueryException(message),
"FORBIDDEN" or "UNAUTHENTICATED" or "UNAUTHORIZED" or "UNAUTHORIZED_ACCESS_ERROR" =>
new SpeckleGraphQLForbiddenException(message),
"FORBIDDEN" or "UNAUTHENTICATED" => new SpeckleGraphQLForbiddenException(message),
"STREAM_NOT_FOUND" => new SpeckleGraphQLStreamNotFoundException(message),
"BAD_USER_INPUT" => new SpeckleGraphQLBadInputException(message),
"INTERNAL_SERVER_ERROR" => new SpeckleGraphQLInternalErrorException(message),
src/Speckle.Sdk/Api/GraphQL/Inputs/FileImportInputs.cs
-12
View File
@@ -2,10 +2,8 @@
public record GenerateFileUploadUrlInput(string projectId, string fileName);
[Obsolete(FileImportInputBase.FILE_IMPORT_DEPRECATION_MESSAGE)]
public record StartFileImportInput(string projectId, string modelId, string fileId, string etag);
[Obsolete(FileImportInputBase.FILE_IMPORT_DEPRECATION_MESSAGE)]
public record FileImportResult(
double durationSeconds,
double downloadDurationSeconds,
@@ -16,23 +14,14 @@ public record FileImportResult(
public abstract class FileImportInputBase
{
internal const string FILE_IMPORT_DEPRECATION_MESSAGE =
"Part of the old API surface and will be removed in the future. Use the new ingestion API instead. Field will be deleted on June 1st, 2026";
[Obsolete(FileImportInputBase.FILE_IMPORT_DEPRECATION_MESSAGE)]
protected FileImportInputBase() { }
public required string projectId { get; init; }
public required string jobId { get; init; }
public required IReadOnlyCollection<string> warnings { get; init; }
[Obsolete(FileImportInputBase.FILE_IMPORT_DEPRECATION_MESSAGE)]
public required FileImportResult result { get; init; }
}
#pragma warning disable CA1822 //Mark members as static
[Obsolete(FILE_IMPORT_DEPRECATION_MESSAGE)]
public sealed class FileImportSuccessInput() : FileImportInputBase()
{
public const string TYPE_STATUS = "success";
@@ -40,7 +29,6 @@ public sealed class FileImportSuccessInput() : FileImportInputBase()
public string status => TYPE_STATUS;
}
[Obsolete(FILE_IMPORT_DEPRECATION_MESSAGE)]
public sealed class FileImportErrorInput() : FileImportInputBase()
{
public const string TYPE_STATUS = "error";
src/Speckle.Sdk/Api/GraphQL/Inputs/ModelIngestionInputs.cs
-71
View File
@@ -1,71 +0,0 @@
using Speckle.Newtonsoft.Json;
using Speckle.Sdk.Api.GraphQL.Enums;
namespace Speckle.Sdk.Api.GraphQL.Inputs;
public record SourceDataInput(
string sourceApplicationSlug,
string sourceApplicationVersion,
string? fileName,
long? fileSizeBytes
);
public record ModelIngestionCreateInput(
string modelId,
string projectId,
string progressMessage,
SourceDataInput sourceData,
int? maxIdleTimeoutSeconds = null
);
public record ModelIngestionUpdateInput(string ingestionId, string projectId, string progressMessage, double? progress);
public record ModelIngestionSuccessInput(
string ingestionId,
string projectId,
string rootObjectId,
string? versionMessage
);
public record ModelIngestionFailedInput(
string ingestionId,
string projectId,
string errorReason,
string? errorStacktrace
)
{
public static ModelIngestionFailedInput FromException(string ingestionId, string projectId, Exception ex)
{
return new ModelIngestionFailedInput(ingestionId, projectId, ex.Message, ex.ToString());
}
}
public record ModelIngestionCancelledInput(string ingestionId, string projectId, string cancellationMessage);
public record ModelIngestionStartProcessingInput(
string ingestionId,
string projectId,
string progressMessage,
SourceDataInput sourceData
);
public record ModelIngestionRequeueInput(string ingestionId, string projectId, string progressMessage);
public record ProjectModelIngestionSubscriptionInput(
string projectId,
ModelIngestionReference ingestionReference,
[property: JsonIgnore] ProjectModelIngestionUpdatedMessageType messageType
)
{
// The Newtonsoft serializer is setup to handle SCREAMING_CASE enums.
// But the API requires the enum to look exactly like they are
[JsonProperty(nameof(messageType))]
public string serializedType => messageType.ToString();
}
/// <remarks>
/// <c>@oneOf</c> i.e. server expects <b>either</b> <paramref name="ingestionId"/> or <paramref name="modelId"/>, but not both.
/// </remarks>
/// <param name="ingestionId"></param>
/// <param name="modelId"></param>
public record ModelIngestionReference(string? ingestionId, string? modelId);
src/Speckle.Sdk/Api/GraphQL/Inputs/VersionInputs.cs
+1 -7
View File
@@ -1,6 +1,4 @@
using Version = Speckle.Sdk.Api.GraphQL.Models.Version;
namespace Speckle.Sdk.Api.GraphQL.Inputs;
namespace Speckle.Sdk.Api.GraphQL.Inputs;
public record UpdateVersionInput(string versionId, string projectId, string? message);
@@ -18,10 +16,6 @@ public record CreateVersionInput(
IReadOnlyList<string>? parents = null
);
/// <param name="versionId"></param>
/// <param name="projectId"></param>
/// <param name="sourceApplication">IMPORTANT: this is meant to be the slug of the application that has done the receiving, not to be confused with <see cref="Version.sourceApplication"/></param>
/// <param name="message"></param>
public record MarkReceivedVersionInput(
string versionId,
string projectId,
src/Speckle.Sdk/Api/GraphQL/Models/Comment.cs
+1 -1
View File
@@ -14,7 +14,7 @@ public sealed class Comment
public string rawText { get; init; }
public ResourceCollection<Comment> replies { get; init; }
public CommentReplyAuthorCollection replyAuthors { get; init; }
public List<ResourceIdentifier> resources { get; init; } //todo: add resourceIds/baseResourceIds
public List<ResourceIdentifier> resources { get; init; }
public string? screenshot { get; init; }
public DateTime updatedAt { get; init; }
public DateTime? viewedAt { get; init; }
src/Speckle.Sdk/Api/GraphQL/Models/ModelIngestion.cs
-13
View File
@@ -1,13 +0,0 @@
namespace Speckle.Sdk.Api.GraphQL.Models;
public sealed class ModelIngestion
{
public required string id { get; init; }
public required DateTime createdAt { get; init; }
public required DateTime updatedAt { get; init; }
public required string modelId { get; init; }
public required string projectId { get; init; }
public required string userId { get; init; }
public required bool cancellationRequested { get; init; }
public required ModelIngestionStatusData statusData { get; init; }
}
src/Speckle.Sdk/Api/GraphQL/Models/ModelIngestionStatusData.cs
-10
View File
@@ -1,10 +0,0 @@
using Speckle.Sdk.Api.GraphQL.Enums;
namespace Speckle.Sdk.Api.GraphQL.Models;
public sealed class ModelIngestionStatusData
{
public required ModelIngestionStatus status { get; init; }
public required string? progressMessage { get; init; }
public required string? versionId { get; init; }
}
src/Speckle.Sdk/Api/GraphQL/Models/ModelPermissionChecks.cs
-8
View File
@@ -1,8 +0,0 @@
namespace Speckle.Sdk.Api.GraphQL.Models;
public sealed class ModelPermissionChecks
{
public PermissionCheckResult canUpdate { get; init; }
public PermissionCheckResult canDelete { get; init; }
public PermissionCheckResult canCreateVersion { get; init; }
}
src/Speckle.Sdk/Api/GraphQL/Models/PendingStreamCollaborator.cs
+1 -1
View File
@@ -10,7 +10,7 @@ public sealed class PendingStreamCollaborator
public string projectName { get; init; }
public string title { get; init; }
public string role { get; init; }
public LimitedUser? invitedBy { get; init; }
public LimitedUser invitedBy { get; init; }
public LimitedUser? user { get; init; }
public string? token { get; init; }
}
src/Speckle.Sdk/Api/GraphQL/Models/ProjectPermissionChecks.cs
-2
View File
@@ -5,7 +5,5 @@ public sealed class ProjectPermissionChecks
public PermissionCheckResult canCreateModel { get; init; }
public PermissionCheckResult canDelete { get; init; }
public PermissionCheckResult canLoad { get; init; }
[Obsolete("Use ModelPermissionChecks.CanCreateVersion instead", true)]
public PermissionCheckResult canPublish { get; init; }
}
src/Speckle.Sdk/Api/GraphQL/Models/ServerInfo.cs
-3
View File
@@ -19,9 +19,6 @@ public sealed class ServerInfo
[Obsolete("Don't use")]
public bool frontend2 { get; set; } = true;
/// <summary>
/// The URL that should be used to talk with the server
/// </summary>
/// <remarks>
/// This field is not returned from the GQL API,
/// it should be populated after construction.
src/Speckle.Sdk/Api/GraphQL/Models/SubscriptionMessages.cs
+15 -24
View File
@@ -6,10 +6,10 @@ namespace Speckle.Sdk.Api.GraphQL.Models;
public sealed class UserProjectsUpdatedMessage : EventArgs
{
[JsonRequired]
public required string id { get; init; }
public string id { get; init; }
[JsonRequired]
public required UserProjectsUpdatedMessageType type { get; init; }
public UserProjectsUpdatedMessageType type { get; init; }
public Project? project { get; init; }
}
@@ -17,10 +17,10 @@ public sealed class UserProjectsUpdatedMessage : EventArgs
public sealed class ProjectCommentsUpdatedMessage : EventArgs
{
[JsonRequired]
public required string id { get; init; }
public string id { get; init; }
[JsonRequired]
public required ProjectCommentsUpdatedMessageType type { get; init; }
public ProjectCommentsUpdatedMessageType type { get; init; }
public Comment? comment { get; init; }
}
@@ -28,10 +28,10 @@ public sealed class ProjectCommentsUpdatedMessage : EventArgs
public sealed class ProjectFileImportUpdatedMessage : EventArgs
{
[JsonRequired]
public required string id { get; init; }
public string id { get; init; }
[JsonRequired]
public required ProjectFileImportUpdatedMessageType type { get; init; }
public ProjectFileImportUpdatedMessageType type { get; init; }
public FileUpload? upload { get; init; }
}
@@ -39,10 +39,10 @@ public sealed class ProjectFileImportUpdatedMessage : EventArgs
public sealed class ProjectModelsUpdatedMessage : EventArgs
{
[JsonRequired]
public required string id { get; init; }
public string id { get; init; }
[JsonRequired]
public required ProjectModelsUpdatedMessageType type { get; init; }
public ProjectModelsUpdatedMessageType type { get; init; }
public Model? model { get; init; }
}
@@ -50,10 +50,10 @@ public sealed class ProjectModelsUpdatedMessage : EventArgs
public sealed class ProjectPendingModelsUpdatedMessage : EventArgs
{
[JsonRequired]
public required string id { get; init; }
public string id { get; init; }
[JsonRequired]
public required ProjectPendingModelsUpdatedMessageType type { get; init; }
public ProjectPendingModelsUpdatedMessageType type { get; init; }
public FileUpload? model { get; init; }
}
@@ -61,10 +61,10 @@ public sealed class ProjectPendingModelsUpdatedMessage : EventArgs
public sealed class ProjectUpdatedMessage : EventArgs
{
[JsonRequired]
public required string id { get; init; }
public string id { get; init; }
[JsonRequired]
public required ProjectUpdatedMessageType type { get; init; }
public ProjectUpdatedMessageType type { get; init; }
public Project? project { get; init; }
}
@@ -72,22 +72,13 @@ public sealed class ProjectUpdatedMessage : EventArgs
public sealed class ProjectVersionsUpdatedMessage : EventArgs
{
[JsonRequired]
public required string id { get; init; }
public string id { get; init; }
[JsonRequired]
public required ProjectVersionsUpdatedMessageType type { get; init; }
public ProjectVersionsUpdatedMessageType type { get; init; }
[JsonRequired]
public required string modelId { get; init; }
public string modelId { get; init; }
public Version? version { get; init; }
}
public sealed class ProjectModelIngestionUpdatedMessage : EventArgs
{
[JsonRequired]
public required ModelIngestion modelIngestion { get; init; }
[JsonRequired]
public required ProjectModelIngestionUpdatedMessageType type { get; init; }
}
src/Speckle.Sdk/Api/GraphQL/Resources/ActiveUserResource.cs
+5
View File
@@ -397,6 +397,11 @@ public sealed class ActiveUserResource
authorized
message
}
canPublish {
code
authorized
message
}
}
}
}
src/Speckle.Sdk/Api/GraphQL/Resources/FileImportResource.cs
+2 -8
View File
@@ -29,10 +29,8 @@ public sealed class FileImportResource : IDisposable
/// <remarks>
/// Only use this if you are writing a file importer, that is responsible for
/// processing file import jobs.
/// Only works on servers version >=2.25.8 but from 3.0.7 onwards has been deprecated and replaced by model ingestion api
/// see <see cref="ModelIngestionResource.Complete"/>
/// Only works on servers version >=2.25.8
/// </remarks>
[Obsolete(FileImportInputBase.FILE_IMPORT_DEPRECATION_MESSAGE)]
public async Task<bool> FinishFileImportJob(FileImportInputBase input, CancellationToken cancellationToken)
{
//language=graphql
@@ -59,11 +57,7 @@ public sealed class FileImportResource : IDisposable
/// <param name="cancellationToken"></param>
/// <returns></returns>
/// <inheritdoc cref="ISpeckleGraphQLClient.ExecuteGraphQLRequest{T}"/>
/// <remarks>
/// Only works on servers version >=2.25.8 but from 3.0.7 onwards has been deprecated and replaced by model ingestion api
/// see <see cref="ModelIngestionResource.StartProcessing"/>
/// </remarks>
[Obsolete(FileImportInputBase.FILE_IMPORT_DEPRECATION_MESSAGE)]
/// <remarks>Only works on servers version >=2.25.8</remarks>
public async Task<FileImport> StartFileImportJob(
StartFileImportInput input,
CancellationToken cancellationToken = default
src/Speckle.Sdk/Api/GraphQL/Resources/ModelIngestionResource.cs
-506
View File
@@ -1,506 +0,0 @@
using GraphQL;
using Speckle.Sdk.Api.GraphQL.Inputs;
using Speckle.Sdk.Api.GraphQL.Models;
using Speckle.Sdk.Api.GraphQL.Models.Responses;
namespace Speckle.Sdk.Api.GraphQL.Resources;
/// <remarks>
/// Model Ingestion API is available for server versions <c>3.0.3</c> and above
/// </remarks>
public sealed class ModelIngestionResource
{
private readonly ISpeckleGraphQLClient _client;
internal ModelIngestionResource(ISpeckleGraphQLClient client)
{
_client = client;
}
/// <summary>
/// Create a new model ingestion
/// </summary>
/// <remarks>
/// The model ingestion created will have a <c>processing</c> state (not <c>queued</c>). This mutation is designed to be used
/// by client/connectors that are immediately processing
/// Model Ingestion API is available for server versions <c>3.0.3</c> and above
/// </remarks>
/// <param name="input"></param>
/// <param name="cancellationToken"></param>
/// <returns></returns>
/// <inheritdoc cref="ISpeckleGraphQLClient.ExecuteGraphQLRequest{T}"/>
public async Task<ModelIngestion> Create(
ModelIngestionCreateInput input,
CancellationToken cancellationToken = default
)
{
//language=graphql
const string QUERY = """
mutation IngestionCreate($input: ModelIngestionCreateInput!) {
data: projectMutations {
data: modelIngestionMutations {
data: create(input: $input) {
id
createdAt
updatedAt
modelId
projectId
userId
cancellationRequested
statusData {
... on HasModelIngestionStatus {
status
}
... on HasProgressMessage {
progressMessage
}
}
}
}
}
}
""";
GraphQLRequest request = new() { Query = QUERY, Variables = new { input } };
var res = await _client
.ExecuteGraphQLRequest<RequiredResponse<RequiredResponse<RequiredResponse<ModelIngestion>>>>(
request,
cancellationToken
)
.ConfigureAwait(false);
return res.data.data.data;
}
/// <remarks>
/// Model Ingestion API is available for server versions <c>3.0.3</c> and above
/// </remarks>
/// <param name="modelIngestionId"></param>
/// <param name="projectId"></param>
/// <param name="cancellationToken"></param>
/// <returns></returns>
/// <inheritdoc cref="ISpeckleGraphQLClient.ExecuteGraphQLRequest{T}"/>
public async Task<ModelIngestion> Get(
string modelIngestionId,
string projectId,
CancellationToken cancellationToken = default
)
{
//language=graphql
const string QUERY = """
query Query($projectId: String!, $modelIngestionId: ID!) {
data:project(id: $projectId) {
data:ingestion(id: $modelIngestionId) {
id
createdAt
updatedAt
modelId
projectId
userId
cancellationRequested
statusData {
... on HasModelIngestionStatus {
status
}
... on HasProgressMessage {
progressMessage
}
... on ModelIngestionSuccessStatus
{
versionId
}
}
}
}
}
""";
GraphQLRequest request = new() { Query = QUERY, Variables = new { projectId, modelIngestionId } };
var res = await _client
.ExecuteGraphQLRequest<RequiredResponse<RequiredResponse<ModelIngestion>>>(request, cancellationToken)
.ConfigureAwait(false);
return res.data.data;
}
/// <summary>
/// For File Import / Cloud integrations only
/// </summary>
/// <remarks>
/// Model Ingestion API is available for server versions <c>3.0.3</c> and above
/// </remarks>
/// <param name="input"></param>
/// <param name="cancellationToken"></param>
/// <returns></returns>
/// <inheritdoc cref="ISpeckleGraphQLClient.ExecuteGraphQLRequest{T}"/>
public async Task<ModelIngestion> StartProcessing(
ModelIngestionStartProcessingInput input,
CancellationToken cancellationToken = default
)
{
//language=graphql
const string QUERY = """
mutation IngestionStartProcessing($input: ModelIngestionStartProcessingInput!) {
data: projectMutations {
data: modelIngestionMutations {
data: startProcessing(input: $input) {
id
createdAt
updatedAt
modelId
projectId
userId
cancellationRequested
statusData {
... on HasModelIngestionStatus {
status
}
... on HasProgressMessage {
progressMessage
}
}
}
}
}
}
""";
GraphQLRequest request = new() { Query = QUERY, Variables = new { input } };
var res = await _client
.ExecuteGraphQLRequest<RequiredResponse<RequiredResponse<RequiredResponse<ModelIngestion>>>>(
request,
cancellationToken
)
.ConfigureAwait(false);
return res.data.data.data;
}
/// <summary>
/// For File Import / Cloud integrations only
/// </summary>
/// <remarks>
/// Model Ingestion API is available for server versions <c>3.0.3</c> and above
/// </remarks>
/// <param name="input"></param>
/// <param name="cancellationToken"></param>
/// <returns></returns>
/// <inheritdoc cref="ISpeckleGraphQLClient.ExecuteGraphQLRequest{T}"/>
public async Task<ModelIngestion> Requeue(
ModelIngestionRequeueInput input,
CancellationToken cancellationToken = default
)
{
//language=graphql
const string QUERY = """
mutation IngestionStartProcessing($input: ModelIngestionRequeueInput!) {
data: projectMutations {
data: modelIngestionMutations {
data: requeue(input: $input) {
id
createdAt
updatedAt
modelId
projectId
userId
cancellationRequested
statusData {
... on HasModelIngestionStatus {
status
}
... on HasProgressMessage {
progressMessage
}
}
}
}
}
}
""";
GraphQLRequest request = new() { Query = QUERY, Variables = new { input } };
var res = await _client
.ExecuteGraphQLRequest<RequiredResponse<RequiredResponse<RequiredResponse<ModelIngestion>>>>(
request,
cancellationToken
)
.ConfigureAwait(false);
return res.data.data.data;
}
/// <remarks>
/// Model Ingestion API is available for server versions <c>3.0.3</c> and above
/// </remarks>
/// <param name="input"></param>
/// <param name="cancellationToken"></param>
/// <returns></returns>
/// <inheritdoc cref="ISpeckleGraphQLClient.ExecuteGraphQLRequest{T}"/>
public async Task<ModelIngestion> UpdateProgress(
ModelIngestionUpdateInput input,
CancellationToken cancellationToken = default
)
{
//language=graphql
const string QUERY = """
mutation IngestionUpdateProgress(
$input: ModelIngestionUpdateInput!
) {
data: projectMutations {
data: modelIngestionMutations {
data: updateProgress(input: $input) {
id
createdAt
updatedAt
modelId
projectId
userId
cancellationRequested
statusData {
... on HasModelIngestionStatus {
status
}
... on HasProgressMessage {
progressMessage
}
}
}
}
}
}
""";
GraphQLRequest request = new() { Query = QUERY, Variables = new { input } };
var res = await _client
.ExecuteGraphQLRequest<RequiredResponse<RequiredResponse<RequiredResponse<ModelIngestion>>>>(
request,
cancellationToken
)
.ConfigureAwait(false);
return res.data.data.data;
}
/// <summary>
/// Request that the server completes the ingestion by creating a version
/// If successful, the job will be in a terminal "successful" state.
/// </summary>
/// <remarks>
/// Model Ingestion API is available for server versions <c>3.0.3</c> and above
/// </remarks>
/// <seealso cref="FailWithError"/>
/// <seealso cref="FailWithCancel"/>
/// <param name="input"></param>
/// <param name="cancellationToken"></param>
/// <returns>The version id</returns>
/// <inheritdoc cref="ISpeckleGraphQLClient.ExecuteGraphQLRequest{T}"/>
public async Task<string> Complete(ModelIngestionSuccessInput input, CancellationToken cancellationToken = default)
{
//language=graphql
const string QUERY = """
mutation IngestionComplete($input: ModelIngestionSuccessInput!) {
data: projectMutations {
data: modelIngestionMutations {
data: completeWithVersion(input: $input) {
data:statusData {
... on ModelIngestionSuccessStatus {
data:versionId
}
}
}
}
}
}
""";
GraphQLRequest request = new() { Query = QUERY, Variables = new { input } };
var res = await _client
.ExecuteGraphQLRequest<
RequiredResponse<RequiredResponse<RequiredResponse<RequiredResponse<RequiredResponse<string>>>>>
>(request, cancellationToken)
.ConfigureAwait(false);
return res.data.data.data.data.data;
}
/// <summary>
/// Fail the job with an error.
/// </summary>
/// <remarks>
/// For requested user cancellation, use <see cref="FailWithCancel"/> instead<br/>
/// Model Ingestion API is available for server versions <c>3.0.3</c> and above
/// </remarks>
/// <seealso cref="FailWithCancel"/>
/// <seealso cref="Complete"/>
/// <param name="input"></param>
/// <param name="cancellationToken"></param>
/// <returns></returns>
/// <inheritdoc cref="ISpeckleGraphQLClient.ExecuteGraphQLRequest{T}"/>
public async Task<ModelIngestion> FailWithError(
ModelIngestionFailedInput input,
CancellationToken cancellationToken = default
)
{
//language=graphql
const string QUERY = """
mutation IngestionFailWithError($input: ModelIngestionFailedInput!) {
data: projectMutations {
data: modelIngestionMutations {
data: failWithError(input: $input) {
id
createdAt
updatedAt
modelId
projectId
userId
cancellationRequested
statusData {
... on HasModelIngestionStatus {
status
}
... on HasProgressMessage {
progressMessage
}
}
}
}
}
}
""";
GraphQLRequest request = new() { Query = QUERY, Variables = new { input } };
var res = await _client
.ExecuteGraphQLRequest<RequiredResponse<RequiredResponse<RequiredResponse<ModelIngestion>>>>(
request,
cancellationToken
)
.ConfigureAwait(false);
return res.data.data.data;
}
/// <summary>
/// Fail the ingestion with a <c>canceled</c> status.
/// This should only be done if the user has explicitly requested cancellation
/// Other forms of cancellation use <see cref="FailWithError"/>.
/// The ingestion should then enter a terminal "canceled" state.<br/>
/// Model Ingestion API is available for server versions <c>3.0.3</c> and above
/// </summary>
/// <seealso cref="FailWithError"/>
/// <seealso cref="Complete"/>
/// <param name="input"></param>
/// <param name="cancellationToken"></param>
/// <returns></returns>
/// <inheritdoc cref="ISpeckleGraphQLClient.ExecuteGraphQLRequest{T}"/>
public async Task<ModelIngestion> FailWithCancel(
ModelIngestionCancelledInput input,
CancellationToken cancellationToken = default
)
{
//language=graphql
const string QUERY = """
mutation IngestionFailWithCancel($input: ModelIngestionCancelledInput!) {
data: projectMutations {
data: modelIngestionMutations {
data: failWithCancel(input: $input) {
id
createdAt
updatedAt
modelId
projectId
userId
cancellationRequested
statusData {
... on HasModelIngestionStatus {
status
}
... on HasProgressMessage {
progressMessage
}
}
}
}
}
}
""";
GraphQLRequest request = new() { Query = QUERY, Variables = new { input } };
var res = await _client
.ExecuteGraphQLRequest<RequiredResponse<RequiredResponse<RequiredResponse<ModelIngestion>>>>(
request,
cancellationToken
)
.ConfigureAwait(false);
return res.data.data.data;
}
/// <summary>
/// Request that the <see cref="ModelIngestion"/> is canceled.
/// </summary>
/// <remarks>
/// Note simply calling this mutation does not imediatly cancel, it doesn't even guarantee it will be canceled at all.
/// It's up to the client to observe this cancellation request
/// via <see cref="SubscriptionResource.CreateProjectModelIngestionCancellationRequestedSubscription"/>
/// and report it as canceled via <see cref="ModelIngestionResource.FailWithCancel"/>
/// See "cooperative cancellation pattern"<br/>
/// Model Ingestion API is available for server versions <c>3.0.3</c> and above
/// </remarks>
/// <seealso cref="FailWithError"/>
/// <seealso cref="Complete"/>
/// <param name="input"></param>
/// <param name="cancellationToken"></param>
/// <returns></returns>
/// <inheritdoc cref="ISpeckleGraphQLClient.ExecuteGraphQLRequest{T}"/>
public async Task<ModelIngestion> RequestCancellation(
ModelIngestionCancelledInput input,
CancellationToken cancellationToken = default
)
{
//language=graphql
const string QUERY = """
mutation IngestionRequestCancellation($input: ModelIngestionRequestCancellationInput!) {
data: projectMutations {
data: modelIngestionMutations {
data: requestCancellation (input: $input) {
id
createdAt
updatedAt
modelId
projectId
userId
cancellationRequested
statusData {
... on HasModelIngestionStatus {
status
}
... on HasProgressMessage {
progressMessage
}
}
}
}
}
}
""";
GraphQLRequest request = new() { Query = QUERY, Variables = new { input } };
var res = await _client
.ExecuteGraphQLRequest<RequiredResponse<RequiredResponse<RequiredResponse<ModelIngestion>>>>(
request,
cancellationToken
)
.ConfigureAwait(false);
return res.data.data.data;
}
}
src/Speckle.Sdk/Api/GraphQL/Resources/ModelResource.cs
-84
View File
@@ -312,88 +312,4 @@ public sealed class ModelResource
return res.data.data;
}
/// <param name="projectId"></param>
/// <param name="cancellationToken"></param>
/// <returns></returns>
/// <inheritdoc cref="ISpeckleGraphQLClient.ExecuteGraphQLRequest{T}"/>
public async Task<ModelPermissionChecks> GetPermissions(
string projectId,
string modelId,
CancellationToken cancellationToken = default
)
{
//language=graphql
const string QUERY = """
query ModelPermissions($projectId: String!, $modelId: String!) {
data:project(id: $projectId) {
data:model(id: $modelId) {
data:permissions {
canUpdate {
authorized
code
message
}
canDelete {
authorized
code
message
}
canCreateVersion {
authorized
code
message
}
}
}
}
}
""";
GraphQLRequest request = new() { Query = QUERY, Variables = new { projectId, modelId } };
var response = await _client
.ExecuteGraphQLRequest<RequiredResponse<RequiredResponse<RequiredResponse<ModelPermissionChecks>>>>(
request,
cancellationToken
)
.ConfigureAwait(false);
return response.data.data.data;
}
/// <param name="projectId"></param>
/// <param name="cancellationToken"></param>
/// <returns></returns>
/// <exception cref="SpeckleGraphQLBadInputException">server versions &lt;3.0.11 do not have <c>canCreateIngestion</c> and will throw this exception</exception>
/// <inheritdoc cref="ISpeckleGraphQLClient.ExecuteGraphQLRequest{T}"/>
public async Task<PermissionCheckResult> CanCreateModelIngestion(
string projectId,
string modelId,
CancellationToken cancellationToken = default
)
{
//language=graphql
const string QUERY = """
query ModelPermissions($projectId: String!, $modelId: String!) {
data:project(id: $projectId) {
data:model(id: $modelId) {
data:permissions {
data:canCreateIngestion {
authorized
code
message
}
}
}
}
}
""";
GraphQLRequest request = new() { Query = QUERY, Variables = new { projectId, modelId } };
var response = await _client
.ExecuteGraphQLRequest<
RequiredResponse<RequiredResponse<RequiredResponse<RequiredResponse<PermissionCheckResult>>>>
>(request, cancellationToken)
.ConfigureAwait(false);
return response.data.data.data.data;
}
}
src/Speckle.Sdk/Api/GraphQL/Resources/SubscriptionResource.cs
-61
View File
@@ -1,5 +1,4 @@
using GraphQL;
using Speckle.Sdk.Api.GraphQL.Enums;
using Speckle.Sdk.Api.GraphQL.Inputs;
using Speckle.Sdk.Api.GraphQL.Models;
using Speckle.Sdk.Api.GraphQL.Models.Responses;
@@ -213,66 +212,6 @@ public sealed class SubscriptionResource : IDisposable
return subscription;
}
/// <summary>Subscribe to a cancellation request being made for a Model Ingestion</summary>
/// <remarks><inheritdoc cref="CreateUserProjectsUpdatedSubscription"/></remarks>
/// <inheritdoc cref="ISpeckleGraphQLClient.SubscribeTo{T}"/>
public Subscription<ProjectModelIngestionUpdatedMessage> CreateProjectModelIngestionUpdatedSubscription(
ProjectModelIngestionSubscriptionInput input
)
{
//language=graphql
const string QUERY = """
subscription IngestionUpdated($input: ProjectModelIngestionSubscriptionInput!) {
data: projectModelIngestionUpdated(input: $input) {
modelIngestion {
id
createdAt
updatedAt
modelId
projectId
userId
cancellationRequested
statusData {
... on HasModelIngestionStatus {
status
}
... on HasProgressMessage {
progressMessage
}
... on ModelIngestionSuccessStatus
{
versionId
}
}
}
type
}
}
""";
GraphQLRequest request = new() { Query = QUERY, Variables = new { input } };
Subscription<ProjectModelIngestionUpdatedMessage> subscription = new(_client, request);
_subscriptions.Add(subscription);
return subscription;
}
/// <summary>Subscribe to a cancellation request being made for a Model Ingestion</summary>
/// <remarks><inheritdoc cref="CreateUserProjectsUpdatedSubscription"/></remarks>
/// <inheritdoc cref="ISpeckleGraphQLClient.SubscribeTo{T}"/>
public Subscription<ProjectModelIngestionUpdatedMessage> CreateProjectModelIngestionCancellationRequestedSubscription(
string ingestionId,
string projectId
)
{
return CreateProjectModelIngestionUpdatedSubscription(
new ProjectModelIngestionSubscriptionInput(
projectId,
new(ingestionId, null),
ProjectModelIngestionUpdatedMessageType.cancellationRequested
)
);
}
public void Dispose()
{
foreach (var subscription in _subscriptions)
src/Speckle.Sdk/Api/GraphQL/Resources/graphql.config.yml
+1 -1
View File
@@ -1,2 +1,2 @@
schema: https://latest.speckle.systems/graphql
schema: https://app.speckle.systems/graphql
documents: '**/*.graphql'
src/Speckle.Sdk/Api/Operations/Operations.Receive.cs
+2 -11
View File
@@ -2,7 +2,6 @@ using Microsoft.Extensions.Logging;
using Speckle.Sdk.Logging;
using Speckle.Sdk.Models;
using Speckle.Sdk.Serialisation;
using Speckle.Sdk.Serialisation.V2.Receive;
using Speckle.Sdk.Transports;
namespace Speckle.Sdk.Api;
@@ -16,7 +15,6 @@ public partial class Operations
/// <exception cref="ArgumentException">No transports were specified</exception>
/// <exception cref="ArgumentNullException">The <paramref name="objectId"/> was <see langword="null"/></exception>
/// <exception cref="SpeckleException">Serialization or Send operation was unsuccessful</exception>
/// <exception cref="HttpRequestException">HTTP layer errors</exception>
/// <exception cref="OperationCanceledException">The <paramref name="cancellationToken"/> requested cancellation</exception>
public async Task<Base> Receive2(
Uri url,
@@ -24,8 +22,7 @@ public partial class Operations
string objectId,
string? authorizationToken,
IProgress<ProgressArgs>? onProgressAction,
CancellationToken cancellationToken,
DeserializeProcessOptions? options = null
CancellationToken cancellationToken
)
{
using var receiveActivity = activityFactory.Start("Operations.Receive");
@@ -39,8 +36,7 @@ public partial class Operations
streamId,
authorizationToken,
onProgressAction,
cancellationToken,
options
cancellationToken
);
try
{
@@ -48,11 +44,6 @@ public partial class Operations
receiveActivity?.SetStatus(SdkActivityStatusCode.Ok);
return result;
}
catch (OperationCanceledException)
{
//this is handled by the caller
throw;
}
catch (Exception ex)
{
receiveActivity?.SetStatus(SdkActivityStatusCode.Error);
src/Speckle.Sdk/Api/Operations/Operations.Send.cs
+2 -4
View File
@@ -25,8 +25,7 @@ public partial class Operations
string? authorizationToken,
Base value,
IProgress<ProgressArgs>? onProgressAction,
CancellationToken cancellationToken,
SerializeProcessOptions? options = null
CancellationToken cancellationToken
)
{
using var receiveActivity = activityFactory.Start("Operations.Send");
@@ -39,8 +38,7 @@ public partial class Operations
streamId,
authorizationToken,
onProgressAction,
cancellationToken,
options
cancellationToken
);
try
{
src/Speckle.Sdk/Credentials/Account.cs
+31 -9
View File
@@ -1,8 +1,11 @@
using System.Runtime.InteropServices;
using Speckle.Sdk.Api.GraphQL.Models;
using Speckle.Sdk.Common;
namespace Speckle.Sdk.Credentials;
[ClassInterface(ClassInterfaceType.AutoDual)]
[ComVisible(true)]
public class Account : IEquatable<Account>
{
private string _id;
@@ -34,8 +37,6 @@ public class Account : IEquatable<Account>
public string? refreshToken { get; set; }
public bool isDefault { get; set; }
[Obsolete("Not used in v3")]
public bool isOnline { get; set; } = true;
public ServerInfo serverInfo { get; set; }
@@ -58,20 +59,15 @@ public class Account : IEquatable<Account>
#region public methods
/// <remarks>The logic is aligned with <c>distinct_id</c> mixpanel property</remarks>
/// <exception cref="ArgumentNullException">Thrown if <see name="userInfo.email"/> was <see langword="null"/></exception>
public string GetHashedEmail()
{
string email = userInfo.email.NotNull();
string email = userInfo?.email ?? "unknown";
return "@" + Md5.GetString(email).ToUpperInvariant();
}
/// <remarks>The logic is aligned with <c>server</c> mixpanel property</remarks>
/// <exception cref="ArgumentNullException">Thrown if <see name="serverInfo.url"/> was <see langword="null"/></exception>
public string GetHashedServer()
{
string url = serverInfo.url.NotNull();
string url = serverInfo?.url ?? AccountManager.DEFAULT_SERVER_URL;
return Md5.GetString(CleanURL(url)).ToUpperInvariant();
}
@@ -100,4 +96,30 @@ public class Account : IEquatable<Account>
}
#endregion
/// <summary>
/// Retrieves the local identifier for the current user.
/// </summary>
/// <returns>
/// Returns a <see cref="Uri"/> object representing the local identifier for the current user.
/// The local identifier is created by appending the user ID as a query parameter to the server URL.
/// </returns>
/// <remarks>
/// Notice that the generated Uri is not intended to be used as a functioning Uri, but rather as a
/// unique identifier for a specific account in a local environment. The format of the Uri, containing a query parameter with the user ID,
/// serves this specific purpose. Therefore, it should not be used for forming network requests or
/// expecting it to lead to an actual webpage. The primary intent of this Uri is for unique identification in a Uri format.
/// </remarks>
/// <example>
/// This sample shows how to call the GetLocalIdentifier method.
/// <code>
/// Uri localIdentifier = GetLocalIdentifier();
/// Console.WriteLine(localIdentifier);
/// </code>
/// For a fictional `User ID: 123` and `Server: https://speckle.xyz`, the output might look like this:
/// <code>
/// https://speckle.xyz?id=123
/// </code>
/// </example>
internal Uri GetLocalIdentifier() => new($"{serverInfo.url}?id={userInfo.id}");
}
src/Speckle.Sdk/Credentials/AccountManager.cs
+538 -112
View File
@@ -1,35 +1,145 @@
using System.Diagnostics;
using System.Net;
using System.Net.Http.Headers;
using System.Security.Cryptography;
using System.Text;
using System.Text.RegularExpressions;
using GraphQL;
using GraphQL.Client.Http;
using Microsoft.Extensions.Logging;
using Speckle.InterfaceGenerator;
using Speckle.Newtonsoft.Json;
using Speckle.Sdk.Api.GraphQL;
using Speckle.Sdk.Api.GraphQL.Models;
using Speckle.Sdk.Api.GraphQL.Models.Responses;
using Speckle.Sdk.Common;
using Speckle.Sdk.Helpers;
using Speckle.Sdk.Logging;
using Speckle.Sdk.SQLite;
using Stream = System.IO.Stream;
namespace Speckle.Sdk.Credentials;
public partial interface IAccountManager : IDisposable;
/// <summary>
/// Manages <see cref="Account"/> data in the local sqlite account store
/// Manage accounts locally for desktop applications.
/// </summary>
[GenerateAutoInterface]
public sealed class AccountManager(
ISpeckleApplication application,
ILogger<AccountManager> logger,
IGraphQLClientFactory graphQLClientFactory,
ISpeckleHttp speckleHttp,
IAccountFactory accountFactory,
IAuthFlow authFlow,
ISqLiteJsonCacheManagerFactory sqLiteJsonCacheManagerFactory
) : IAccountManager
{
public const string DEFAULT_SERVER_URL = "https://app.speckle.systems";
private readonly ISqLiteJsonCacheManager _accountStorage = sqLiteJsonCacheManagerFactory.CreateForUser("Accounts");
private static volatile bool s_isAddingAccount;
private readonly ISqLiteJsonCacheManager _accountAddLockStorage = sqLiteJsonCacheManagerFactory.CreateForUser(
"AccountAddFlow"
);
[AutoInterfaceIgnore]
public void Dispose()
{
_accountStorage.Dispose();
_accountAddLockStorage.Dispose();
}
/// <summary>
/// Gets the basic information about a server.
/// </summary>
/// <param name="server">Server Information</param>
/// <returns></returns>
/// <exception cref="GraphQLHttpRequestException">Request failed on the HTTP layer (received a non-successful response code)</exception>
/// <exception cref="AggregateException"><inheritdoc cref="GraphQLErrorHandler.EnsureGraphQLSuccess(IGraphQLResponse)"/></exception>
public async Task<ServerInfo> GetServerInfo(Uri server, CancellationToken cancellationToken = default)
{
using var gqlClient = graphQLClientFactory.CreateGraphQLClient(server, null);
//lang=graphql
const string QUERY_STRING = "query { serverInfo { name company migration { movedFrom movedTo } } }";
var request = new GraphQLRequest { Query = QUERY_STRING };
var response = await gqlClient.SendQueryAsync<ServerInfoResponse>(request, cancellationToken).ConfigureAwait(false);
response.EnsureGraphQLSuccess();
ServerInfo serverInfo = response.Data.serverInfo;
serverInfo.url = server.ToString().TrimEnd('/');
return response.Data.serverInfo;
}
/// <summary>
/// Gets basic user information given a token and a server.
/// </summary>
/// <param name="token"></param>
/// <param name="server">Server URL</param>
/// <returns></returns>
/// <exception cref="GraphQLHttpRequestException">Request failed on the HTTP layer (received a non-successful response code)</exception>
/// <exception cref="AggregateException"><inheritdoc cref="GraphQLErrorHandler.EnsureGraphQLSuccess(IGraphQLResponse)"/></exception>
public async Task<UserInfo> GetUserInfo(string token, Uri server, CancellationToken cancellationToken = default)
{
using var gqlClient = graphQLClientFactory.CreateGraphQLClient(server, token);
//language=graphql
const string QUERY = """
query {
data:activeUser {
name
email
id
company
}
}
""";
var request = new GraphQLRequest { Query = QUERY };
var response = await gqlClient
.SendQueryAsync<RequiredResponse<UserInfo>>(request, cancellationToken)
.ConfigureAwait(false);
response.EnsureGraphQLSuccess();
return response.Data.data;
}
/// <summary>
/// The Default Server URL for authentication, can be overridden by placing a file with the alternatrive url in the Speckle folder or with an ENV_VAR
/// </summary>
public Uri GetDefaultServerUrl()
{
var customServerUrl = "";
// first mechanism, check for local file
var customServerFile = Path.Combine(SpecklePathProvider.UserSpeckleFolderPath, "server");
if (File.Exists(customServerFile))
{
customServerUrl = File.ReadAllText(customServerFile);
}
// second mechanism, check ENV VAR
var customServerEnvVar = Environment.GetEnvironmentVariable("SPECKLE_SERVER");
if (!string.IsNullOrEmpty(customServerEnvVar))
{
customServerUrl = customServerEnvVar;
}
if (!string.IsNullOrEmpty(customServerUrl))
{
if (Uri.TryCreate(customServerUrl, UriKind.Absolute, out Uri? url))
{
return url;
}
}
return new Uri(DEFAULT_SERVER_URL);
}
/// <param name="id">The Id of the account to fetch</param>
@@ -41,6 +151,37 @@ public sealed class AccountManager(
?? throw new SpeckleAccountManagerException($"Account {id} not found");
}
/// <summary>
/// Upgrades an account from the account.serverInfo.movedFrom account to the account.serverInfo.movedTo account
/// </summary>
/// <param name="id">Id of the account to upgrade</param>
public void UpgradeAccount(string id)
{
Account account = GetAccount(id);
if (account.serverInfo.migration?.movedTo is not Uri upgradeUri)
{
throw new SpeckleAccountManagerException(
$"Server with url {account.serverInfo.url} does not have information about the upgraded server"
);
}
account.serverInfo.migration.movedTo = null;
account.serverInfo.migration.movedFrom = new Uri(account.serverInfo.url);
account.serverInfo.url = upgradeUri.ToString().TrimEnd('/');
// setting the id to null will force it to be recreated
account.id = null!; //TODO this is gross so remove when id is nullable
RemoveAccount(id);
_accountStorage.UpdateObject(account.id.NotNull(), JsonConvert.SerializeObject(account));
}
public IEnumerable<Account> GetAccounts(string serverUrl)
{
return GetAccounts(new Uri(serverUrl));
}
/// <summary>
/// Returns all unique accounts matching the serverUrl provided. If an account exists on more than one server,
/// typically because it has been migrated, then only the upgraded account (and therefore server) are returned.
@@ -104,6 +245,7 @@ public sealed class AccountManager(
static bool IsInvalid(Account ac) => ac.userInfo == null || ac.serverInfo == null;
var sqlAccounts = _accountStorage.GetAllObjects().Select(x => JsonConvert.DeserializeObject<Account>(x.Json));
var localAccounts = GetLocalAccounts();
foreach (var acc in sqlAccounts)
{
@@ -117,55 +259,119 @@ public sealed class AccountManager(
yield return acc;
}
}
}
/// <summary>
/// Refetches all local accounts (in local db), including <see cref="ServerInfo"/> and <see cref="UserInfo"/>.
/// If the <see cref="Account.token"/> looks to be expired, this function will also attempt to use the <see cref="Account.refreshToken"/> to refresh it.
/// Will write the changes to the local accounts db
/// </summary>
/// <seealso cref="UpdateAccount"/>
/// <param name="cancellationToken"></param>
/// <exception cref="AggregateException"></exception>
public async Task UpdateAccount(Account account, CancellationToken cancellationToken = default)
{
string oldAccountId = account.id;
await UpdateAccountInMemory(account, cancellationToken).ConfigureAwait(false);
if (oldAccountId != account.id)
foreach (var acc in localAccounts)
{
// ID may have changed, e.g. users email changed, or server url migrated
_accountStorage.DeleteObject(oldAccountId);
yield return acc;
}
_accountStorage.UpdateObject(account.id, JsonConvert.SerializeObject(account));
}
/// <summary>
/// Refetches the <paramref name="account"/> information, including <see cref="ServerInfo"/> and <see cref="UserInfo"/>
///
/// Will only mutate <paramref name="account"/> in memory only, and only if successful.
/// Gets the local accounts
/// These are accounts not handled by Manager and are stored in json format in a local directory
/// </summary>
/// <returns></returns>
private IList<Account> GetLocalAccounts()
{
var accountsDir = SpecklePathProvider.AccountsFolderPath;
if (!Directory.Exists(accountsDir))
{
return Array.Empty<Account>();
}
var accounts = new List<Account>();
string[] files = Directory.GetFiles(accountsDir, "*.json", SearchOption.AllDirectories);
foreach (var file in files)
{
try
{
var json = File.ReadAllText(file);
Account? account = JsonConvert.DeserializeObject<Account>(json);
if (
account is not null
&& !string.IsNullOrEmpty(account.token)
&& !string.IsNullOrEmpty(account.userInfo.id)
&& !string.IsNullOrEmpty(account.userInfo.email)
&& !string.IsNullOrEmpty(account.userInfo.name)
&& !string.IsNullOrEmpty(account.serverInfo.url)
&& !string.IsNullOrEmpty(account.serverInfo.name)
)
{
accounts.Add(account);
}
}
catch (Exception ex) when (!ex.IsFatal())
{
logger.LogWarning(ex, "Failed to load json account at {filePath}", file);
}
}
return accounts;
}
/// <summary>
/// Refetches user and server info for each account
/// </summary>
/// <param name="app"> It is defaultAppId in the server. By default it is "sca" to not break existing parts that this function involves.</param>
/// <returns></returns>
public async Task UpdateAccounts(CancellationToken ct = default, string app = "sca")
{
// need to ToList() the GetAccounts call or the UpdateObject call at the end of this method
// will not work because sqlite does not support concurrent db calls
foreach (var account in GetAccounts().ToList())
{
try
{
Uri url = new(account.serverInfo.url);
var userServerInfo = await accountFactory.GetUserServerInfo(url, account.token, ct).ConfigureAwait(false);
//the token has expired
//TODO: once we get a token expired exception from the server use that instead
if (userServerInfo.activeUser == null || userServerInfo.serverInfo == null)
{
// We were initially was handling refresh token here bc quite a while ago server was returning null
// for activeUser and serverInfo instead of throwing exception. In short, our logic moved into catch block to cover both.
throw new SpeckleException("Token is expired");
}
account.isOnline = true;
account.userInfo = userServerInfo.activeUser;
account.serverInfo = userServerInfo.serverInfo;
}
catch (OperationCanceledException)
{
throw;
}
catch (Exception ex) when (!ex.IsFatal())
{
await RefreshAndSetAccountToken(account, app).ConfigureAwait(false);
}
ct.ThrowIfCancellationRequested();
_accountStorage.UpdateObject(account.id, JsonConvert.SerializeObject(account));
}
}
/// <summary>
/// Mutates the account with new tokens.
/// </summary>
/// <seealso cref="UpdateAccount"/>
/// <param name="account"></param>
/// <param name="cancellationToken"></param>
/// <exception cref="GraphQLHttpRequestException"></exception>
public async Task UpdateAccountInMemory(Account account, CancellationToken cancellationToken = default)
/// <param name="app"></param>
private async Task RefreshAndSetAccountToken(Account account, string app)
{
Uri url = account.serverInfo.migration?.movedTo ?? new(account.serverInfo.url);
ActiveUserServerInfoResponse userServerInfo = await accountFactory
.GetUserServerInfo(url, account.token, cancellationToken)
.ConfigureAwait(false);
if (userServerInfo.activeUser == null)
try
{
throw new SpeckleException("GraphQL response indicated that the ActiveUser could not be found");
Uri url = new(account.serverInfo.url);
var tokenResponse = await GetRefreshedToken(account.refreshToken, url, app).ConfigureAwait(false);
account.token = tokenResponse.token;
account.refreshToken = tokenResponse.refreshToken;
account.isOnline = true;
}
catch (Exception ex) when (!ex.IsFatal())
{
account.isOnline = false;
}
account.userInfo = userServerInfo.activeUser;
account.serverInfo = userServerInfo.serverInfo;
//This is a bit gross, since id is not marked nullable
//but this will force re-generate the id (e.g. if the user's email, or servers url has changed)
account.id = null!;
}
/// <summary>
@@ -206,103 +412,323 @@ public sealed class AccountManager(
}
/// <summary>
/// Adds an account to local storage by prompting the user to log in via their browser.
/// Retrieves the local identifier for the specified account.
/// </summary>
/// <example>
/// <code>
/// Account account = await AuthenticateAccount(new Uri("https://app.speckle.systems"), TimeSpan.FromMinutes(1));
/// </code>
/// </example>
/// <param name="serverUrl"></param>
/// <param name="timeout">Timeout for user to auth with browser, recommend 1 min timeout</param>
/// <param name="cancellationToken"></param>
/// <returns></returns>
public async Task<Account> AuthenticateAccount(Uri serverUrl, TimeSpan timeout, CancellationToken cancellationToken)
/// <param name="account">The account for which to retrieve the local identifier.</param>
/// <returns>The local identifier for the specified account in the form of "SERVER_URL?u=USER_ID".</returns>
/// <remarks>
/// <inheritdoc cref="Account.GetLocalIdentifier"/>
/// </remarks>
public Uri? GetLocalIdentifierForAccount(Account account)
{
logger.LogDebug("Starting to add account for {ServerUrl}", serverUrl);
var identifier = account.GetLocalIdentifier();
TokenExchangeResponse tokenResponse = await authFlow
.TriggerAuthFlowWithTimeout(serverUrl, AuthApp.ConnectorsV3, timeout, cancellationToken)
.ConfigureAwait(false);
// Validate account is stored locally
var searchResult = GetAccountForLocalIdentifier(identifier);
return await CreateAndAddAccount(serverUrl, tokenResponse, cancellationToken).ConfigureAwait(false);
return searchResult == null ? null : identifier;
}
public async Task<Account> CreateAndAddAccount(
Uri serverUrl,
TokenExchangeResponse tokenResponse,
CancellationToken cancellationToken
)
public async Task<UserInfo> Validate(Account account)
{
var account = await accountFactory
.CreateAccount(serverUrl, tokenResponse.token, tokenResponse.refreshToken, cancellationToken)
.ConfigureAwait(false);
account.isDefault = !GetAccounts().Any();
_accountStorage.SaveObject(account.id, JsonConvert.SerializeObject(account));
logger.LogInformation("Successfully authenticated account {AccountId} for {ServerUrl}", account.id, serverUrl);
return account;
Uri server = new(account.serverInfo.url);
return await GetUserInfo(account.token, server).ConfigureAwait(false);
}
/// <summary>
/// The Default Server URL for authentication, can be overridden by placing a file with the alternative url in the Speckle folder or with an ENV_VAR
/// Gets the account that corresponds to the given local identifier.
/// </summary>
[Obsolete("Unused")]
public Uri GetDefaultServerUrl()
/// <param name="localIdentifier">The local identifier of the account.</param>
/// <returns>The account that matches the local identifier, or null if no match is found.</returns>
public Account? GetAccountForLocalIdentifier(Uri localIdentifier)
{
var customServerUrl = "";
// first mechanism, check for local file
var customServerFile = Path.Combine(SpecklePathProvider.UserSpeckleFolderPath, "server");
if (File.Exists(customServerFile))
{
customServerUrl = File.ReadAllText(customServerFile);
}
// second mechanism, check ENV VAR
var customServerEnvVar = Environment.GetEnvironmentVariable("SPECKLE_SERVER");
if (!string.IsNullOrEmpty(customServerEnvVar))
{
customServerUrl = customServerEnvVar;
}
if (!string.IsNullOrEmpty(customServerUrl))
{
if (Uri.TryCreate(customServerUrl, UriKind.Absolute, out Uri? url))
var searchResult = GetAccounts()
.FirstOrDefault(acc =>
{
return url;
var id = acc.GetLocalIdentifier();
return id == localIdentifier;
});
return searchResult;
}
private Uri EnsureCorrectServerUrl(Uri? server)
{
var localUrl = server;
if (localUrl == null)
{
localUrl = GetDefaultServerUrl();
logger.LogDebug("The provided server url was null or empty. Changed to the default url {serverUrl}", localUrl);
}
return localUrl;
}
private void EnsureGetAccessCodeFlowIsSupported()
{
if (!HttpListener.IsSupported)
{
logger.LogError("HttpListener not supported");
throw new PlatformNotSupportedException("Your operating system is not supported");
}
}
private async Task<string> GetAccessCode(Uri server, string challenge, TimeSpan timeout)
{
EnsureGetAccessCodeFlowIsSupported();
logger.LogDebug("Starting auth process for {server}/authn/verify/sca/{challenge}", server, challenge);
var accessCode = "";
Process.Start(new ProcessStartInfo($"{server}/authn/verify/sca/{challenge}") { UseShellExecute = true });
var task = Task.Run(() =>
{
using var listener = new HttpListener();
var localUrl = "http://localhost:29363/";
listener.Prefixes.Add(localUrl);
listener.Start();
logger.LogDebug("Listening for auth redirects on {localUrl}", localUrl);
// Note: The GetContext method blocks while waiting for a request.
HttpListenerContext context = listener.GetContext();
HttpListenerRequest request = context.Request;
HttpListenerResponse response = context.Response;
accessCode = request.QueryString["access_code"];
logger.LogDebug("Got access code {accessCode}", accessCode);
string message =
accessCode != null
? "Success!<br/><br/>You can close this window now.<script>window.close();</script>"
: "Oups, something went wrong...!";
var responseString =
$"<HTML><BODY Style='background: linear-gradient(to top right, #ffffff, #c8e8ff); font-family: Roboto, sans-serif; font-size: 2rem; font-weight: 500; text-align: center;'><br/>{message}</BODY></HTML>";
byte[] buffer = Encoding.UTF8.GetBytes(responseString);
response.ContentLength64 = buffer.Length;
Stream output = response.OutputStream;
output.Write(buffer, 0, buffer.Length);
output.Close();
logger.LogDebug("Processed finished processing the access code");
listener.Stop();
listener.Close();
});
var completedTask = await Task.WhenAny(task, Task.Delay(timeout)).ConfigureAwait(false);
// this is means the task timed out
if (completedTask != task)
{
logger.LogWarning(
"Local auth flow failed to complete within the timeout window. Access code is {accessCode}",
accessCode
);
throw new AuthFlowException("Local auth flow failed to complete within the timeout window");
}
if (task.IsFaulted && task.Exception is not null)
{
logger.LogError(
task.Exception,
"Getting access code flow failed with {exceptionMessage}",
task.Exception.Message
);
throw new AuthFlowException($"Auth flow failed: {task.Exception.Message}", task.Exception);
}
// task completed within timeout
logger.LogInformation(
"Local auth flow completed successfully within the timeout window. Access code is {accessCode}",
accessCode
);
return accessCode;
}
private async Task<Account> CreateAccount(string accessCode, string challenge, Uri server)
{
try
{
var tokenResponse = await GetToken(accessCode, challenge, server).ConfigureAwait(false);
var account = await accountFactory
.CreateAccount(server, tokenResponse.token, tokenResponse.refreshToken)
.ConfigureAwait(false);
account.isDefault = !GetAccounts().Any();
logger.LogInformation("Successfully created account for {serverUrl}", server);
return account;
}
catch (Exception ex) when (!ex.IsFatal())
{
throw new SpeckleAccountManagerException("Failed to create account from access code and challenge", ex);
}
}
private void TryLockAccountAddFlow(TimeSpan timespan)
{
// use a static variable to quickly
// prevent launching this flow multiple times
if (s_isAddingAccount)
{
// this should probably throw with an error message
throw new SpeckleAccountFlowLockedException("The account add flow is already launched.");
}
// this uses the SQLite transport to store locks
var lockIds = _accountAddLockStorage.GetAllObjects().Select(x => x.Id).OrderByDescending(d => d).ToList();
var now = DateTime.Now;
foreach (var l in lockIds)
{
var lockArray = l.Split('@');
var lockName = lockArray.Length == 2 ? lockArray[0] : "the other app";
var lockTime =
lockArray.Length == 2
? DateTime.ParseExact(lockArray[1], "o", null)
: DateTime.ParseExact(lockArray[0], "o", null);
if (lockTime > now)
{
var lockString = string.Format("{0:mm} minutes {0:ss} seconds", lockTime - now);
throw new SpeckleAccountFlowLockedException(
$"The account add flow was already started in {lockName}, retry in {lockString}"
);
}
}
return new Uri(DEFAULT_SERVER_URL);
var lockId = application.ApplicationAndVersion + "@" + DateTime.Now.Add(timespan).ToString("o");
// using the lock release time as an id and value
// for ease of deletion and retrieval
_accountAddLockStorage.SaveObject(lockId, lockId);
s_isAddingAccount = true;
}
[Obsolete("Use Uri overload")]
public IEnumerable<Account> GetAccounts(string serverUrl)
private void UnlockAccountAddFlow()
{
return GetAccounts(new Uri(serverUrl));
s_isAddingAccount = false;
// make sure all old locks are removed
foreach (var (id, _) in _accountAddLockStorage.GetAllObjects())
{
_accountAddLockStorage.DeleteObject(id);
}
}
[Obsolete("Use UpdateAccount instead for more control over error handling", true)]
public Task UpdateAccounts(CancellationToken ct = default, string app = "sca") => throw new NotImplementedException();
/// <summary>
/// Adds an account by propting the user to log in via a web flow
/// </summary>
/// <param name="server">Server to use to add the account, if not provied the default Server will be used</param>
/// <returns></returns>
public async Task AddAccount(Uri? server = null)
{
logger.LogDebug("Starting to add account for {serverUrl}", server);
[Obsolete("Use UpdateAccount instead", true)]
public void UpgradeAccount(string id) => throw new NotImplementedException();
server = EnsureCorrectServerUrl(server);
[Obsolete($"Use {nameof(AuthenticateAccount)} instead", true)]
public Task AddAccount(Uri? server = null) => throw new NotImplementedException();
// locking for 1 minute
var timeout = TimeSpan.FromMinutes(1);
// this is not part of the try finally block
// we do not want to clean up the existing locks
TryLockAccountAddFlow(timeout);
var challenge = GenerateChallenge();
[Obsolete("Use serverInfo stored on a client instead", true)]
public Task<ServerInfo> GetServerInfo(Uri server, CancellationToken cancellationToken = default) =>
throw new NotImplementedException();
try
{
string accessCode = await GetAccessCode(server, challenge, timeout).ConfigureAwait(false);
if (string.IsNullOrEmpty(accessCode))
{
throw new SpeckleAccountManagerException("Access code is invalid");
}
[Obsolete("Use userInfo stored on a client instead", true)]
public Task<UserInfo> GetUserInfo(string token, Uri server, CancellationToken cancellationToken = default) =>
throw new NotImplementedException();
var account = await CreateAccount(accessCode, challenge, server).ConfigureAwait(false);
[Obsolete("Accounts must now be stored in sqlite db, no more json workaround", true)]
public IList<Account> GetLocalAccounts() => throw new NotImplementedException();
//if the account already exists it will not be added again
_accountStorage.SaveObject(account.id, JsonConvert.SerializeObject(account));
logger.LogDebug("Finished adding account {accountId} for {serverUrl}", account.id, server);
}
catch (SpeckleAccountManagerException ex)
{
logger.LogCritical(ex, "Failed to add account: {exceptionMessage}", ex.Message);
// rethrowing any known errors
throw;
}
catch (Exception ex) when (!ex.IsFatal())
{
logger.LogCritical(ex, "Failed to add account: {exceptionMessage}", ex.Message);
throw new SpeckleAccountManagerException($"Failed to add account: {ex.Message}", ex);
}
finally
{
UnlockAccountAddFlow();
}
}
[Obsolete("Use UpdateAccount or UpdateAccountInMemory Instead", true)]
public IList<Account> Validate() => throw new NotImplementedException();
private async Task<TokenExchangeResponse> GetToken(string accessCode, string challenge, Uri server)
{
try
{
using var client = speckleHttp.CreateHttpClient();
var body = new
{
appId = "sca",
appSecret = "sca",
accessCode,
challenge,
};
using var content = new StringContent(JsonConvert.SerializeObject(body));
content.Headers.ContentType = new MediaTypeHeaderValue("application/json");
var response = await client.PostAsync(new Uri(server, "/auth/token"), content).ConfigureAwait(false);
return JsonConvert
.DeserializeObject<TokenExchangeResponse>(await response.Content.ReadAsStringAsync().ConfigureAwait(false))
.NotNull();
}
catch (Exception ex) when (!ex.IsFatal())
{
throw new SpeckleException($"Failed to get authentication token from {server}", ex);
}
}
private async Task<TokenExchangeResponse> GetRefreshedToken(string? refreshToken, Uri server, string app = "sca")
{
try
{
using var client = speckleHttp.CreateHttpClient();
var body = new
{
appId = app,
appSecret = app,
refreshToken,
};
using var content = new StringContent(JsonConvert.SerializeObject(body));
content.Headers.ContentType = new MediaTypeHeaderValue("application/json");
var response = await client.PostAsync(new Uri(server, "/auth/token"), content).ConfigureAwait(false);
return JsonConvert
.DeserializeObject<TokenExchangeResponse>(await response.Content.ReadAsStringAsync().ConfigureAwait(false))
.NotNull();
}
catch (Exception ex) when (!ex.IsFatal())
{
throw new SpeckleException($"Failed to get refreshed token from {server}", ex);
}
}
private static string GenerateChallenge()
{
#if NET8_0
byte[] challengeData = RandomNumberGenerator.GetBytes(32);
#else
using RNGCryptoServiceProvider rng = new();
byte[] challengeData = new byte[32];
rng.GetBytes(challengeData);
#endif
//escaped chars like % do not play nice with the server
return Regex.Replace(Convert.ToBase64String(challengeData), @"[^\w\.@-]", "");
}
}
src/Speckle.Sdk/Credentials/AuthApp.cs
-13
View File
@@ -1,13 +0,0 @@
namespace Speckle.Sdk.Credentials;
public readonly record struct AuthApp(string AppId, string AppSecret, Uri CallbackUrl)
{
//These values are defined on the server, and specify the scopes the app is requesting
public static AuthApp ConnectorsV3 { get; } =
new()
{
AppId = "connectrV3",
AppSecret = "connectrV3",
CallbackUrl = new Uri("http://localhost:29355"),
};
}
src/Speckle.Sdk/Credentials/AuthFlow.cs
-330
View File
@@ -1,330 +0,0 @@
using System.Diagnostics;
using System.Diagnostics.Contracts;
using System.Net;
using System.Net.Http.Headers;
using System.Security.Cryptography;
using System.Text;
using Speckle.InterfaceGenerator;
using Speckle.Newtonsoft.Json;
using Speckle.Sdk.Common;
using Speckle.Sdk.Helpers;
using Speckle.Sdk.Logging;
namespace Speckle.Sdk.Credentials;
/// <summary>
/// Authentication flow with the Speckle Server to create a application token for the <c>connectorsV3</c> application
/// Starts the browser based authentication flow where the user's browser will be opened, they'll be asked to
/// confirm permission, then an access code will be given via a <see cref="HttpListener"/> which will be exchanged
/// for a <see cref="TokenExchangeResponse"/>
/// </summary>
/// <remarks>
/// Note, this class is not coupled in any way to <see cref="Account"/>
/// lets keep it that way...
/// See instead <see cref="AccountManager"/>
/// </remarks>
[GenerateAutoInterface]
public sealed class AuthFlow(ISdkActivityFactory activityFactory, ISpeckleHttp speckleHttp) : IAuthFlow
{
private readonly JsonSerializerSettings _serializerSettings = new()
{
MissingMemberHandling = MissingMemberHandling.Error,
NullValueHandling = NullValueHandling.Ignore,
};
public async Task<TokenExchangeResponse> TriggerAuthFlowWithTimeout(
Uri serverUrl,
AuthApp authApp,
TimeSpan timeout,
CancellationToken cancellationToken
)
{
using HttpClient client = speckleHttp.CreateHttpClient();
Uri tokenEndpoint = new(serverUrl, "/oauth/token");
string codeVerifier = GenerateCodeVerifier();
Uri authnVerify;
using var req = await client.GetAsync(tokenEndpoint, cancellationToken).ConfigureAwait(false);
bool useLegacyEndpoint = req.StatusCode != HttpStatusCode.OK;
if (useLegacyEndpoint)
{
string challenge = codeVerifier; // Old endpoint only supports PKCE "plain" method
authnVerify = new($"/authn/verify/{authApp.AppId}/{challenge}", UriKind.Relative);
tokenEndpoint = new(serverUrl, "/auth/token");
}
else
{
string challenge = GenerateCodeChallenge(codeVerifier);
authnVerify = new($"/authn/verify/{authApp.AppId}/{challenge}?code_challenge_method=S256", UriKind.Relative);
}
Uri endpoint = new(serverUrl, authnVerify);
_ = Process.Start(new ProcessStartInfo(endpoint.ToString()) { UseShellExecute = true });
string accessCode = await RunListenerWithTimeout(authApp.CallbackUrl, timeout, cancellationToken)
.ConfigureAwait(false);
object body = useLegacyEndpoint
? new
{
appId = authApp.AppId,
appSecret = authApp.AppSecret,
accessCode = accessCode,
challenge = codeVerifier,
}
: new
{
appId = authApp.AppId,
accessCode = accessCode,
codeVerifier = codeVerifier,
};
return await ExchangeAccessCodeForToken(
client,
JsonConvert.SerializeObject(body, _serializerSettings),
tokenEndpoint,
cancellationToken
)
.ConfigureAwait(false);
}
/// <summary>
///
/// </summary>
/// <param name="applicationCallbackUrl"></param>
/// <param name="timeout"></param>
/// <param name="userCancellation"></param>
/// <returns></returns>
/// <exception cref="OperationCanceledException"><paramref name="userCancellation"/> requested cancel</exception>
/// <exception cref="TimeoutException">timeout was reached</exception>
public async Task<string> RunListenerWithTimeout(
Uri applicationCallbackUrl,
TimeSpan timeout,
CancellationToken userCancellation
)
{
using CancellationTokenSource cancelOnTimeout = new(timeout);
using CancellationTokenSource linkedSource = CancellationTokenSource.CreateLinkedTokenSource(
cancelOnTimeout.Token,
userCancellation
);
try
{
using var activity = activityFactory.Start("Listening for authflow access code");
return await RunListener(applicationCallbackUrl, linkedSource.Token).ConfigureAwait(false);
}
catch (OperationCanceledException) when (userCancellation.IsCancellationRequested)
{
throw;
}
catch (OperationCanceledException ex) when (cancelOnTimeout.IsCancellationRequested)
{
throw new TimeoutException($"Auth flow was cancelled after {timeout:g} timeout", ex);
}
}
/// <summary>
///
/// </summary>
/// <param name="refreshToken"></param>
/// <param name="serverUrl"></param>
/// <param name="authApp">Auth app, needs to match the app that generated the refresh token originally</param>
/// <param name="cancellationToken"></param>
/// <exception cref="HttpRequestException">HTTP exceptions</exception>
/// <exception cref="JsonSerializationException">Server response was invalid or partial</exception>
/// <exception cref="ArgumentOutOfRangeException ">Invalid <paramref name="serverUrl"/> (must be absolute url)</exception>
/// <exception cref="OperationCanceledException"><paramref name="cancellationToken"/> requested cancel</exception>
/// <returns></returns>
public async Task<TokenExchangeResponse> GetRefreshedToken(
string? refreshToken,
Uri serverUrl,
AuthApp authApp,
CancellationToken cancellationToken
)
{
using var client = speckleHttp.CreateHttpClient();
var body = new
{
appId = authApp.AppId,
appSecret = authApp.AppSecret,
refreshToken = refreshToken,
};
using var content = new StringContent(JsonConvert.SerializeObject(body, _serializerSettings));
content.Headers.ContentType = new MediaTypeHeaderValue("application/json");
var response = await client
.PostAsync(new Uri(serverUrl, "/auth/token"), content, cancellationToken)
.ConfigureAwait(false);
response.EnsureSuccessStatusCode();
#if NET8_0_OR_GREATER
string read = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false);
#else
string read = await response.Content.ReadAsStringAsync().ConfigureAwait(false);
#endif
return JsonConvert.DeserializeObject<TokenExchangeResponse>(read, _serializerSettings).NotNull();
}
private static async Task<HttpListenerContext> GetContext(HttpListener listener, CancellationToken cancellationToken)
{
//GetContextAsync doesn't support cancellation, so we have to do this song and dance...
Task timeoutTask = Task.Delay(Timeout.Infinite, cancellationToken);
Task<HttpListenerContext> getContextTask = listener.GetContextAsync();
Task completed = await Task.WhenAny(getContextTask, timeoutTask).ConfigureAwait(false);
if (completed == getContextTask)
{
return getContextTask.Result;
}
cancellationToken.ThrowIfCancellationRequested();
throw new InvalidOperationException("Cancellation should have thrown, this shouldn't be possible");
}
public static async Task<string> RunListener(Uri localUrl, CancellationToken cancellationToken)
{
using HttpListener listener = new();
listener.Prefixes.Add(localUrl.ToString());
listener.Start();
HttpListenerContext context = await GetContext(listener, cancellationToken).ConfigureAwait(false);
HttpListenerRequest request = context.Request;
using HttpListenerResponse response = context.Response;
string? accessCode = request.QueryString["access_code"];
string? denied = request.QueryString["denied"];
bool isDenied = denied == "true";
if (isDenied)
{
//lang=html
WriteResponse(
"""
<h1>Denied!</h1>
<br/><br/>
Please close this window and return to your Speckle Connector.
"""
);
throw new AuthFlowException("Authentication flow was denied"); //denied presumably by the user
}
else if (accessCode != null)
{
//lang=html
WriteResponse(
"""
<h1>Success!</h1>
<br/><br/>
Your Speckle Connector is now authorized
<br/><br/>
You may now close this window and return to your Speckle Connector
"""
);
return accessCode;
}
else
{
//lang=html
WriteResponse(
"""
<h1>Failed!</h1>
<br/><br/>
Something went wrong trying to authorize your Speckle Connector
<br/><br/>
Please close this window and try again from your Speckle Connector.
"""
);
throw new AuthFlowException("Failed to receive access code");
}
void WriteResponse(string message)
{
//lang=html
string responseString = $"""
<HTML>
<BODY Style='background: #FAFAFAFF; font-family: Inter, Roboto, sans-serif; font-size: 1rem; font-weight: 500; text-align: center;'>
<br/>
{message}
</BODY>
</HTML>
""";
byte[] buffer = Encoding.UTF8.GetBytes(responseString);
response.ContentLength64 = buffer.Length;
response.OutputStream.Write(buffer, 0, buffer.Length);
}
}
private async Task<TokenExchangeResponse> ExchangeAccessCodeForToken(
HttpClient client,
string jsonContent,
Uri tokenEndpoint,
CancellationToken cancellationToken
)
{
using StringContent content = new(jsonContent);
content.Headers.ContentType = new MediaTypeHeaderValue("application/json");
using HttpResponseMessage response = await client
.PostAsync(tokenEndpoint, content, cancellationToken)
.ConfigureAwait(false);
response.EnsureSuccessStatusCode();
#if NET8_0_OR_GREATER
string read = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false);
#else
string read = await response.Content.ReadAsStringAsync().ConfigureAwait(false);
#endif
return JsonConvert.DeserializeObject<TokenExchangeResponse>(read, _serializerSettings).NotNull();
}
[Pure]
public static string GenerateCodeVerifier()
{
#if NET8_0_OR_GREATER
Span<byte> codeVerifierData = stackalloc byte[32];
RandomNumberGenerator.Fill(codeVerifierData);
#else
using RNGCryptoServiceProvider rng = new();
byte[] codeVerifierData = new byte[32];
rng.GetBytes(codeVerifierData);
#endif
return Base64UrlEncode(codeVerifierData);
}
[Pure]
public static string GenerateCodeChallenge(string codeVerifier)
{
#if NET8_0_OR_GREATER
int byteCount = Encoding.UTF8.GetByteCount(codeVerifier.AsSpan());
Span<byte> codeVerifierBytes = stackalloc byte[byteCount];
Encoding.UTF8.GetBytes(codeVerifier, codeVerifierBytes);
Span<byte> challengeData = stackalloc byte[SHA256.HashSizeInBytes];
SHA256.HashData(codeVerifierBytes, challengeData);
#else
byte[] codeVerifierBytes = Encoding.UTF8.GetBytes(codeVerifier);
using SHA256 hash = SHA256.Create();
byte[] challengeData = hash.ComputeHash(codeVerifierBytes);
#endif
return Base64UrlEncode(challengeData);
}
[Pure]
private static string Base64UrlEncode(
#if NET8_0_OR_GREATER
ReadOnlySpan<byte> bytes
#else
byte[] bytes
#endif
)
{
// Base64Url is available in .NET 9, or via the Microsoft.Bcl.Memory polyfill
// But for simplicity r.e. dll dependencies, we're doing it the dumb way...
return Convert.ToBase64String(bytes).Replace('+', '-').Replace('/', '_').TrimEnd('=');
}
}
src/Speckle.Sdk/Credentials/AuthFlowException.cs
+14
View File
@@ -0,0 +1,14 @@
namespace Speckle.Sdk.Credentials;
#pragma warning disable CA2237
public sealed class AuthFlowException : Exception
#pragma warning restore CA2237
{
public AuthFlowException(string? message, Exception? innerException)
: base(message, innerException) { }
public AuthFlowException(string? message)
: base(message) { }
public AuthFlowException() { }
}
src/Speckle.Sdk/Credentials/Exceptions.cs
+11 -11
View File
@@ -1,16 +1,5 @@
namespace Speckle.Sdk.Credentials;
public sealed class AuthFlowException : SpeckleException
{
public AuthFlowException(string? message, Exception? innerException)
: base(message, innerException) { }
public AuthFlowException(string? message)
: base(message) { }
public AuthFlowException() { }
}
public class SpeckleAccountManagerException : SpeckleException
{
public SpeckleAccountManagerException(string message)
@@ -21,3 +10,14 @@ public class SpeckleAccountManagerException : SpeckleException
public SpeckleAccountManagerException() { }
}
public class SpeckleAccountFlowLockedException : SpeckleAccountManagerException
{
public SpeckleAccountFlowLockedException(string message)
: base(message) { }
public SpeckleAccountFlowLockedException() { }
public SpeckleAccountFlowLockedException(string message, Exception? innerException)
: base(message, innerException) { }
}
src/Speckle.Sdk/Credentials/Responses.cs
+5 -8
View File
@@ -6,19 +6,16 @@ namespace Speckle.Sdk.Credentials;
internal sealed class ActiveUserServerInfoResponse
{
[property: JsonProperty(Required = Required.AllowNull)]
public required UserInfo? activeUser { get; init; }
public UserInfo? activeUser { get; init; }
[property: JsonProperty(Required = Required.Always)]
public required ServerInfo serverInfo { get; init; }
public ServerInfo serverInfo { get; init; }
}
public sealed class TokenExchangeResponse
internal sealed class TokenExchangeResponse
{
[JsonRequired]
public required string token { get; init; }
[JsonRequired]
public required string refreshToken { get; init; }
public string token { get; init; }
public string refreshToken { get; init; }
}
public sealed class UserInfo
src/Speckle.Sdk/Helpers/BlobApiHelpers.cs
-29
View File
@@ -1,29 +0,0 @@
using System.Net.Http.Headers;
namespace Speckle.Sdk.Helpers;
public static class BlobApiHelpers
{
public static string ParseEtagHeader(HttpResponseHeaders headers)
{
if (!headers.TryGetValues("ETag", out var etagValues))
{
throw new ArgumentException(
"Response does not have an ETag attached to it, cannot use this as an upload",
nameof(headers)
);
}
var etagValuesArray = etagValues.ToArray();
if (etagValuesArray.Length != 1)
{
throw new ArgumentException(
$"Expected Etag header to have a single value but got {etagValuesArray.Length}",
nameof(headers)
);
}
return etagValuesArray[0];
}
}
src/Speckle.Sdk/Helpers/DisposableFile.cs
-34
View File
@@ -1,34 +0,0 @@
using Microsoft.Extensions.Logging;
namespace Speckle.Sdk.Helpers;
/// <summary>
/// <see cref="IDisposable"/> wrapper around the downloaded file to try and delete the file on Dispose
/// </summary>
/// <remarks>
/// We're using a similar pattern in the Rhino File Importer codebase (see <c>ImportJobFile</c>)
/// </remarks>
/// <param name="logger"></param>
/// <param name="file"></param>
public sealed class DisposableFile(FileInfo file, ILogger logger, bool deleteOnDispose = true) : IDisposable
{
public FileInfo FileInfo => file;
public void Dispose()
{
if (!deleteOnDispose)
{
return;
}
try
{
file.Delete();
logger.LogInformation("Cleaned up {File}", file);
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
{
logger.LogWarning(ex, "Failed to clean up {File}", file);
}
}
}
src/Speckle.Sdk/Helpers/StopwatchPollyfills.cs
-21
View File
@@ -1,21 +0,0 @@
using System.Diagnostics;
namespace Speckle.Sdk.Helpers;
public static class StopwatchPolyfills
{
#if !NET7_0_OR_GREATER
private static readonly double s_tickFrequency = (double)TimeSpan.TicksPerSecond / Stopwatch.Frequency;
#endif
public static TimeSpan GetElapsedTime(long startingTimestamp)
{
#if NET7_0_OR_GREATER
return Stopwatch.GetElapsedTime(startingTimestamp);
#else
long elapsedTicks = Stopwatch.GetTimestamp() - startingTimestamp;
return new TimeSpan((long)(elapsedTicks * s_tickFrequency));
#endif
}
}
src/Speckle.Sdk/Host/TypeLoader.cs
+1 -16
View File
@@ -140,22 +140,7 @@ internal static class TypeLoader
return typeof(Base);
}
/// <summary>
/// For testing purposes only
/// </summary>
internal static void ReInitialize(params Assembly[] assemblies)
{
lock (s_availableTypes)
{
Reset();
Load(assemblies);
s_initialized = true;
}
}
/// <summary>
/// For testing purposes only
/// </summary>
//Don't use unless you're testing
public static void Reset()
{
s_availableTypes = new();
src/Speckle.Sdk/Logging/NullActivityFactory.cs
+2 -6
View File
@@ -1,12 +1,8 @@
using Speckle.Connectors.Logging;
namespace Speckle.Sdk.Logging;
namespace Speckle.Sdk.Logging;
public sealed class NullActivityFactory : ISdkActivityFactory
{
public void Dispose() { }
public ISdkActivity? Start(string? name, SdkActivityKind kind, string source) => null;
public ISdkActivity? StartRemote(string traceContext, SdkActivityKind kind, string? name, string source) => null;
public ISdkActivity? Start(string? name = default, string source = "") => null;
}
src/Speckle.Sdk/Logging/SpecklePathProvider.cs
+18 -12
View File
@@ -9,14 +9,12 @@ public static class SpecklePathProvider
{
private const string APPLICATION_NAME = "Speckle";
private const string LOG_FOLDER_NAME = "Logs";
private const string BLOB_FOLDER_NAME = "Blobs";
private const string ACCOUNTS_FOLDER_NAME = "Accounts";
public const string USER_DATA_PATH_ENV_VAR = "SPECKLE_USERDATA_PATH";
private static string? Path => Environment.GetEnvironmentVariable(USER_DATA_PATH_ENV_VAR);
private static string UserDataPathEnvVar => "SPECKLE_USERDATA_PATH";
private static string? Path => Environment.GetEnvironmentVariable(UserDataPathEnvVar);
/// <summary>
/// Get the installation path.
@@ -45,7 +43,7 @@ public static class SpecklePathProvider
/// <see cref="Environment.SpecialFolder.ApplicationData"/> path usually maps to
/// <ul>
/// <li>win: <c>%appdata%/</c></li>
/// <li>MacOS: <c>~/Library/Application Support</c></li>
/// <li>MacOS: <c>~/.config/</c></li>
/// <li>Linux: <c>~/.config/</c></li>
/// </ul>
/// </remarks>
@@ -59,18 +57,29 @@ public static class SpecklePathProvider
return pathOverride;
}
// on desktop linux and macos we use the appdata.
// but we might not have write access to the disk
// so the catch falls back to the user profile
try
{
return Environment.GetFolderPath(
Environment.SpecialFolder.ApplicationData,
// It's not a given that the folder is already there on all OS-es, so we'll create it
// if the folder doesn't exist, we get back an empty string on OSX,
// which in turn, breaks other stuff down the line.
// passing in the Create option ensures that this directory exists,
// which is not a given on all OS-es.
Environment.SpecialFolderOption.Create
);
}
catch (PlatformNotSupportedException)
catch (SystemException ex) when (ex is PlatformNotSupportedException or ArgumentException)
{
// We might not have write access to the disk to create the folder,
// so we'll fall back to the user profile
//Adding this log just so we confidently know which Exception type to catch here.
// TODO: Must re-add log call when (and if) this get's made as a service
//SpeckleLog.Logger.Warning(ex, "Falling back to user profile path");
// on server linux, there might not be a user setup, things can run under root
// in that case, the appdata variable is most probably not set up
// we fall back to the value of the home folder
return Environment.GetFolderPath(Environment.SpecialFolder.UserProfile);
}
}
@@ -87,7 +96,4 @@ public static class SpecklePathProvider
Directory.CreateDirectory(path);
return path;
}
public static string LogFolderPath(string applicationAndVersion) =>
EnsureFolderExists(UserSpeckleFolderPath, LOG_FOLDER_NAME, applicationAndVersion);
}
src/Speckle.Sdk/Models/Collections/RootCollection.cs
-21
View File
@@ -1,21 +0,0 @@
using Speckle.Sdk.Models.Data;
namespace Speckle.Sdk.Models.Collections;
/// <summary>
/// Root collection that represents the top-level commit object.
/// Extends Collection to include model-wide properties that apply to the entire model.
/// </summary>
[SpeckleType("Speckle.Core.Models.Collections.RootCollection")]
public class RootCollection : Collection, IProperties
{
public RootCollection() { }
public RootCollection(string name)
: base(name) { }
/// <summary>
/// Model-wide properties that apply to the entire model.
/// </summary>
public Dictionary<string, object?> properties { get; set; } = new();
}
src/Speckle.Sdk/Models/Data/IProperties.cs
-10
View File
@@ -1,10 +0,0 @@
namespace Speckle.Sdk.Models.Data;
/// <summary>
/// Specifies properties on objects to be used for data-based workflows.
/// Can be applied to both objects and collections.
/// </summary>
public interface IProperties
{
Dictionary<string, object?> properties { get; }
}
src/Speckle.Sdk/Models/DynamicBaseMemberType.cs
+3 -4
View File
@@ -7,12 +7,12 @@ namespace Speckle.Sdk.Models;
public enum DynamicBaseMemberType
{
/// <summary>
/// The typed members of the <see cref="DynamicBase"/> object
/// The typed members of the DynamicBase object
/// </summary>
Instance = 1,
/// <summary>
/// The dynamically added members of the <see cref="DynamicBase"/> object
/// The dynamically added members of the DynamicBase object
/// </summary>
Dynamic = 2,
@@ -22,9 +22,8 @@ public enum DynamicBaseMemberType
Obsolete = 4,
/// <summary>
/// Old feature supported in v2 for grasshopper
/// The typed methods flagged with TODO:
/// </summary>
[Obsolete("Feature no longer supported")]
SchemaComputed = 16,
/// <summary>
src/Speckle.Sdk/Models/GraphTraversal/GraphTraversal.cs
+2 -2
View File
@@ -109,9 +109,9 @@ public abstract class GraphTraversal<T>
break;
case IList list:
{
for (int i = list.Count - 1; i >= 0; i--)
foreach (object? obj in list)
{
TraverseMemberToStack(stack, list[i], memberName, parent);
TraverseMemberToStack(stack, obj, memberName, parent);
}
break;
src/Speckle.Sdk/Pipelines/Progress/AggregateProgress.cs
-12
View File
@@ -1,12 +0,0 @@
namespace Speckle.Sdk.Pipelines.Progress;
public sealed class AggregateProgress<T>(params IProgress<T>[] progresses) : IProgress<T>
{
public void Report(T value)
{
foreach (var progress in progresses)
{
progress.Report(value);
}
}
}
src/Speckle.Sdk/Pipelines/Progress/IngestionProgressManager.cs
-89
View File
@@ -1,89 +0,0 @@
using System.Diagnostics;
using Microsoft.Extensions.Logging;
using Speckle.InterfaceGenerator;
using Speckle.Sdk.Api;
using Speckle.Sdk.Api.GraphQL.Inputs;
using Speckle.Sdk.Api.GraphQL.Models;
using Speckle.Sdk.Helpers;
namespace Speckle.Sdk.Pipelines.Progress;
public partial interface IIngestionProgressManager : IProgress<CardProgress>;
/// <summary>
/// An <see langword="IProgress{IngestionProgressEventArgs}"/> implementation for the entire client side Ingestion progress update reporting
/// Will throttles ingestion progress messages and reports their progress
/// </summary>
/// <remarks>
/// Normally we would pick quite a coarse updateInterval to try and spamming the server (1-5s)
/// </remarks>
[GenerateAutoInterface]
public sealed class IngestionProgressManager(
ILogger<IngestionProgressManager> logger,
IClient speckleClient,
ModelIngestion ingestion,
TimeSpan updateInterval,
CancellationToken cancellationToken
) : IIngestionProgressManager
{
public Task? LastUpdate { get; private set; }
private long _lastUpdatedAt;
private readonly object _lock = new();
[AutoInterfaceIgnore]
public void Report(CardProgress value)
{
cancellationToken.ThrowIfCancellationRequested();
string trimmedMessage;
lock (_lock)
{
if (ShouldIgnoreProgressUpdate())
{
return;
}
_lastUpdatedAt = Stopwatch.GetTimestamp();
trimmedMessage = value.Status.TrimEnd('.');
LastUpdate = speckleClient
.Ingestion.UpdateProgress(
new ModelIngestionUpdateInput(ingestion.id, ingestion.projectId, trimmedMessage, value.Progress),
cancellationToken
)
.ContinueWith(
Continuation,
CancellationToken.None,
TaskContinuationOptions.ExecuteSynchronously,
TaskScheduler.Default
);
}
logger.LogInformation("Progress update {Message} {Progress}", trimmedMessage, value.Progress);
}
/// <returns><see langword="true"/> if the update should be ignored, otherwise <see langword="false"/></returns>
private bool ShouldIgnoreProgressUpdate()
{
if (LastUpdate is not null && !LastUpdate.IsCompleted)
{
return true;
}
TimeSpan msSinceLastUpdate = StopwatchPolyfills.GetElapsedTime(_lastUpdatedAt);
return msSinceLastUpdate < updateInterval;
}
private void Continuation(Task updateTask)
{
// The progress report failed... could be many reasons.
// For now, we're not letting this fail the Ingestion in any way
// we'll log but otherwise let it slide while leaving no unobserved task exceptions
if (updateTask.IsFaulted)
{
logger.LogWarning(updateTask.Exception, "A progress update failed unexpectedly");
}
}
}
src/Speckle.Sdk/Pipelines/Progress/IngestionProgressManagerFactory.cs
-21
View File
@@ -1,21 +0,0 @@
using Microsoft.Extensions.Logging;
using Speckle.InterfaceGenerator;
using Speckle.Sdk.Api;
using Speckle.Sdk.Api.GraphQL.Models;
namespace Speckle.Sdk.Pipelines.Progress;
[GenerateAutoInterface]
public sealed class IngestionProgressManagerFactory(ILogger<IngestionProgressManager> logger)
: IIngestionProgressManagerFactory
{
public IIngestionProgressManager CreateInstance(
IClient speckleClient,
ModelIngestion ingestion,
TimeSpan updateInterval,
CancellationToken cancellationToken
)
{
return new IngestionProgressManager(logger, speckleClient, ingestion, updateInterval, cancellationToken);
}
}
src/Speckle.Sdk/Pipelines/Progress/ProgressArgs.cs
-6
View File
@@ -1,6 +0,0 @@
namespace Speckle.Sdk.Pipelines.Progress;
//TODO: rename PipelineProgressArgs
public readonly record struct CardProgress(string Status, double? Progress);
public readonly record struct StreamProgressArgs(long BytesStreamed, long ExpectedTotalBytes);
src/Speckle.Sdk/Pipelines/Progress/ProgressStream.cs
-103
View File
@@ -1,103 +0,0 @@
using System.Diagnostics.CodeAnalysis;
namespace Speckle.Sdk.Pipelines.Progress;
/// <summary>
/// Wraps <paramref name="innerStream"/> to report streaming progress as bytes are read/written.
/// </summary>
public sealed class ProgressStream(Stream innerStream, IProgress<StreamProgressArgs>? progress = null) : Stream
{
private long _bytesStreamed;
public override bool CanRead => innerStream.CanRead;
public override bool CanSeek => innerStream.CanSeek;
public override bool CanWrite => innerStream.CanWrite;
public override long Length => innerStream.Length;
public override long Position
{
get => innerStream.Position;
set => innerStream.Position = value;
}
public override int Read(byte[] buffer, int offset, int count)
{
int bytesRead = innerStream.Read(buffer, offset, count);
ReportProgress(bytesRead);
return bytesRead;
}
[SuppressMessage(
"Performance",
"CA1835:Prefer the \'Memory\'-based overloads for \'ReadAsync\' and \'WriteAsync\'",
Justification = "Analyser warning forwarded to caller"
)]
public override async Task<int> ReadAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken)
{
int bytesRead = await innerStream.ReadAsync(buffer, offset, count, cancellationToken).ConfigureAwait(false);
ReportProgress(bytesRead);
return bytesRead;
}
#if NET8_0_OR_GREATER
public override async ValueTask<int> ReadAsync(Memory<byte> buffer, CancellationToken cancellationToken = default)
{
int bytesRead = await innerStream.ReadAsync(buffer, cancellationToken).ConfigureAwait(false);
ReportProgress(bytesRead);
return bytesRead;
}
#endif
private void ReportProgress(int newBytesProcessed)
{
_bytesStreamed += newBytesProcessed;
progress?.Report(new(_bytesStreamed, Length));
}
public override void Flush() => innerStream.Flush();
public override Task FlushAsync(CancellationToken cancellationToken) => innerStream.FlushAsync(cancellationToken);
public override long Seek(long offset, SeekOrigin origin) => innerStream.Seek(offset, origin);
public override void SetLength(long value) => throw new NotSupportedException(); //intentionally not supporting, as changing length of stream mid-flight will fuck up progress
public override void Write(byte[] buffer, int offset, int count)
{
innerStream.Write(buffer, offset, count);
ReportProgress(count);
}
[SuppressMessage(
"Performance",
"CA1835:Prefer the \'Memory\'-based overloads for \'ReadAsync\' and \'WriteAsync\'",
Justification = "Analyser warning forwarded to caller"
)]
public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken)
{
await innerStream.WriteAsync(buffer, offset, count, cancellationToken).ConfigureAwait(false);
ReportProgress(count);
}
#if NET6_0_OR_GREATER
public override async ValueTask WriteAsync(ReadOnlyMemory<byte> buffer, CancellationToken cancellationToken = default)
{
await innerStream.WriteAsync(buffer, cancellationToken).ConfigureAwait(false);
ReportProgress(buffer.Length);
}
#endif
protected override void Dispose(bool disposing)
{
innerStream.Dispose();
base.Dispose(disposing);
}
#if NET6_0_OR_GREATER
public override async ValueTask DisposeAsync()
{
await innerStream.DisposeAsync().ConfigureAwait(false);
await base.DisposeAsync().ConfigureAwait(false);
}
#endif
}
src/Speckle.Sdk/Pipelines/Progress/RenderedStreamProgress.cs
-40
View File
@@ -1,40 +0,0 @@
namespace Speckle.Sdk.Pipelines.Progress;
/// <summary>
/// Renders "low level" data stream updates
/// into "high level" <see cref="CardProgress"/> that is expected by Ingestion progress and DUI3
/// </summary>
/// <param name="progress"></param>
public sealed class RenderedStreamProgress(IProgress<CardProgress> progress) : IProgress<StreamProgressArgs>
{
public void Report(StreamProgressArgs value)
{
var (suffix, scaleFactor) = GetFileSizeRendering(value.ExpectedTotalBytes);
progress.Report(
new(
$"Uploading data... ({value.BytesStreamed * scaleFactor:F1}/{value.ExpectedTotalBytes * scaleFactor:F1} {suffix})",
(double)value.BytesStreamed / value.ExpectedTotalBytes
)
);
}
private static readonly string[] s_suffixes = ["B", "KB", "MB", "GB", "TB", "PB"];
internal static (string suffix, double scaleFactor) GetFileSizeRendering(long value)
{
if (value <= 0)
{
return (s_suffixes[0], 1d);
}
for (int i = 0; i < s_suffixes.Length; i++)
{
if (value <= Math.Pow(1024, i + 1))
{
return (s_suffixes[i], 1 / Math.Pow(1024, i));
}
}
throw new ArgumentOutOfRangeException(nameof(value), "Value is too large to convert to a file size");
}
}
src/Speckle.Sdk/Pipelines/Send/DiskStore.cs
-90
View File
@@ -1,90 +0,0 @@
using System.IO.Compression;
using Microsoft.Extensions.Logging;
using Speckle.InterfaceGenerator;
using Speckle.Sdk.Dependencies;
using Speckle.Sdk.Helpers;
using Speckle.Sdk.Logging;
namespace Speckle.Sdk.Pipelines.Send;
[GenerateAutoInterface]
public sealed class DiskStoreFactory(ILogger<DiskStore> logger, ISdkActivityFactory activityFactory) : IDiskStoreFactory
{
public DiskStore CreateInstance(CancellationToken cancellationToken) =>
new(logger, activityFactory, cancellationToken);
}
public sealed class DiskStore
{
private readonly RepackedChannel<UploadItem> _channel;
private readonly Task<DisposableFile> _writeToDiskTask;
private readonly ILogger<DiskStore> _logger;
private readonly ISdkActivityFactory _activityFactory;
private readonly CancellationToken _cancellationToken;
internal DiskStore(
ILogger<DiskStore> logger,
ISdkActivityFactory activityFactory,
CancellationToken cancellationToken
)
{
_logger = logger;
_activityFactory = activityFactory;
_cancellationToken = cancellationToken;
_channel = new RepackedChannel<UploadItem>(1000, true, false);
_writeToDiskTask = Task.Run(WriteFile, cancellationToken);
}
public async Task PushAsync(UploadItem item) =>
await _channel.WriteAsync(item, _cancellationToken).ConfigureAwait(false);
public async Task<DisposableFile> CompleteAsync()
{
using var a = _activityFactory.Start("Waiting for DiskStore to complete");
_channel.CompleteWriter();
return await _writeToDiskTask.ConfigureAwait(false);
}
/// <summary>
/// Reads from the Channel and streams the <see cref="UploadItem"/>s to a temporary file on disk.
/// Will keep reading until <see cref="CompleteAsync"/> is called.
/// </summary>
/// <returns>the file that was written</returns>
private async Task<DisposableFile> WriteFile()
{
string tempFilePath = Path.GetTempFileName();
var tempFile = new DisposableFile(new FileInfo(tempFilePath), _logger);
_logger.LogInformation("Writing temp file to {TempFilePath}", tempFilePath);
try
{
using var fileStream = new FileStream(tempFilePath, FileMode.Create, FileAccess.Write, FileShare.None);
using var gzip = new GZipStream(fileStream, CompressionLevel.Optimal);
using var writer = new StreamWriter(gzip);
await foreach (var item in _channel.ReadAllAsync(_cancellationToken).ConfigureAwait(false))
{
await writer.WriteAsync(item.Id).ConfigureAwait(false);
await writer.WriteAsync('\t').ConfigureAwait(false);
await writer.WriteAsync(item.SpeckleType).ConfigureAwait(false);
await writer.WriteAsync('\t').ConfigureAwait(false);
await writer.WriteAsync(item.Json.Value).ConfigureAwait(false);
await writer.WriteLineAsync().ConfigureAwait(false);
}
#if NET8_0_OR_GREATER
await writer.FlushAsync(_cancellationToken).ConfigureAwait(false);
#else
await writer.FlushAsync().ConfigureAwait(false);
#endif
tempFile.FileInfo.Refresh();
return tempFile;
}
catch
{
tempFile.Dispose();
throw;
}
}
}
src/Speckle.Sdk/Pipelines/Send/SendPipeline.cs
-74
View File
@@ -1,74 +0,0 @@
using Speckle.InterfaceGenerator;
using Speckle.Sdk.Credentials;
using Speckle.Sdk.Helpers;
using Speckle.Sdk.Models;
using Speckle.Sdk.Pipelines.Progress;
namespace Speckle.Sdk.Pipelines.Send;
[GenerateAutoInterface]
public sealed class SendPipelineFactory(IUploaderFactory uploaderFactory, IDiskStoreFactory diskStoreFactory)
: ISendPipelineFactory
{
public SendPipeline CreateInstance(
string projectId,
string ingestionId,
Account account,
IProgress<StreamProgressArgs> uploadProgress,
CancellationToken cancellationToken
)
{
var uploader = uploaderFactory.CreateInstance(projectId, ingestionId, account, uploadProgress, cancellationToken);
var diskStore = diskStoreFactory.CreateInstance(cancellationToken);
return new SendPipeline(uploader, diskStore);
}
}
public sealed class SendPipeline : IDisposable
{
private readonly Serializer _serializer = new();
private readonly Uploader _uploader;
private readonly DiskStore _diskStore;
internal SendPipeline(Uploader uploader, DiskStore diskStore)
{
_uploader = uploader;
_diskStore = diskStore;
}
private UploadItem _lastItem;
public async Task<ObjectReference> Process(Base @base)
{
var results = _serializer.Serialize(@base).ToArray();
var first = results.First();
foreach (var item in results)
{
// we're not doing fire and forget here so that we get the backpressure from the uploader
await _diskStore.PushAsync(item).ConfigureAwait(false);
}
// NOTE: this is important to keep track of. When we serialze an object, we get back a list of objects, with the first one being the original root.
// In the case of the commit root object, this means the last object is not necessarily the root; we therefore need to manually track its existance here
// and ensure it's the last one through in the uploader's stream. See WaitForUpload down below.
_lastItem = first;
return first.Reference;
}
public async Task WaitForUpload()
{
await _diskStore.PushAsync(_lastItem).ConfigureAwait(false);
using DisposableFile tempFile = await _diskStore.CompleteAsync().ConfigureAwait(false);
using Stream fileStreamUpload = new FileStream(
tempFile.FileInfo.FullName,
FileMode.Open,
FileAccess.Read,
FileShare.Read
);
await _uploader.Send(fileStreamUpload).ConfigureAwait(false);
}
public void Dispose() => _uploader.Dispose();
}
src/Speckle.Sdk/Pipelines/Send/Serializer.cs
-351
View File
@@ -1,351 +0,0 @@
using System.Collections;
using System.Drawing;
using System.Globalization;
using System.Reflection;
using Speckle.DoubleNumerics;
using Speckle.Newtonsoft.Json;
using Speckle.Sdk.Dependencies;
using Speckle.Sdk.Helpers;
using Speckle.Sdk.Models;
using Speckle.Sdk.Serialisation;
namespace Speckle.Sdk.Pipelines.Send;
/// <summary>
/// Another serializer, cleaner and meaner. Provides methods for serializing Speckle objects into a format suitable for upload or storage.
/// This class handles the conversion of <see cref="Speckle.Sdk.Models.Base"/> and its derivatives
/// into serialized JSON structures along with associated metadata, closures, and references.
/// <para>Any reference objects coming through are being "passed through" serialized - they do not get double encoded.</para>
/// </summary>
internal sealed class Serializer
{
private readonly record struct PropertyInfo(string Name, object? Value, bool IsDetachable);
public IEnumerable<UploadItem> Serialize(Base root)
{
// Special case: if root is already an ObjectReference, serialize it verbatim
if (root is ObjectReference existingRef)
{
var uploadItem = ReferenceToUploadItem(existingRef);
yield return uploadItem;
yield break;
}
var detachedObjects = new List<(Id, Json, Dictionary<string, int>, Base, string)>();
var rootClosures = new Dictionary<string, int>();
var (rootId, rootJson) = SerializeBase(root, false, rootClosures, detachedObjects);
var rootReference = new ObjectReference
{
referencedId = rootId.Value,
applicationId = root.applicationId,
closure = rootClosures.Count > 0 ? rootClosures : null,
};
yield return new UploadItem(rootId.Value, rootJson, root.speckle_type, rootReference);
foreach (var (id, json, closures, baseObj, speckleType) in detachedObjects)
{
var reference = new ObjectReference
{
referencedId = id.Value,
applicationId = baseObj.applicationId,
closure = closures.Count > 0 ? closures : null,
};
yield return new UploadItem(id.Value, json, speckleType, reference);
}
}
private IEnumerable<PropertyInfo> ExtractProperties(Base baseObj)
{
var typedProperties = baseObj.GetInstanceMembers();
foreach (var prop in typedProperties)
{
if (prop.Name == "id" || prop.Name.StartsWith("__"))
{
continue;
}
if (prop.IsDefined(typeof(JsonIgnoreAttribute), false))
{
continue;
}
var value = prop.GetValue(baseObj);
var isDetachable = prop.GetCustomAttribute<DetachPropertyAttribute>(true)?.Detachable ?? false;
yield return new PropertyInfo(prop.Name, value, isDetachable);
}
foreach (var propName in baseObj.DynamicPropertyKeys)
{
if (propName.StartsWith("__"))
{
continue;
}
var value = baseObj[propName];
#pragma warning disable CA1866
var isDetachable = propName.StartsWith("@");
#pragma warning restore CA1866
yield return new PropertyInfo(propName, value, isDetachable);
}
}
private (Id, Json) SerializeBase(
Base baseObj,
bool forceDetach,
Dictionary<string, int> closures,
List<(Id, Json, Dictionary<string, int>, Base, string)> detachedObjects
)
{
var childClosures = new Dictionary<string, int>();
var sb = Pools.StringBuilders.Get();
try
{
using var stringWriter = new StringWriter(sb);
using var jsonWriter = new JsonTextWriter(stringWriter);
using var idWriter = new SerializerIdWriter(jsonWriter);
idWriter.WriteStartObject();
foreach (var prop in ExtractProperties(baseObj))
{
idWriter.WritePropertyName(prop.Name);
SerializeValue(prop.Value, idWriter, prop.IsDetachable, childClosures, detachedObjects);
}
var (jsonForId, finalWriter) = idWriter.FinishIdWriter();
var id = IdGenerator.ComputeId(jsonForId);
finalWriter.WritePropertyName("id");
finalWriter.WriteValue(id.Value);
baseObj.id = id.Value;
if ((forceDetach || childClosures.Count > 0) && childClosures.Count > 0)
{
finalWriter.WritePropertyName("__closure");
finalWriter.WriteStartObject();
foreach (var kvp in childClosures)
{
finalWriter.WritePropertyName(kvp.Key);
finalWriter.WriteValue(kvp.Value);
}
finalWriter.WriteEndObject();
foreach (var kvp in childClosures)
{
closures[kvp.Key] = closures.TryGetValue(kvp.Key, out var existing) ? existing + kvp.Value : kvp.Value;
}
}
finalWriter.WriteEndObject();
finalWriter.Flush();
var json = new Json(stringWriter.ToString());
return (id, json);
}
finally
{
Pools.StringBuilders.Return(sb);
}
}
private void SerializeValue(
object? value,
JsonWriter writer,
bool isDetachable,
Dictionary<string, int> closures,
List<(Id, Json, Dictionary<string, int>, Base, string)> detachedObjects
)
{
switch (value)
{
case Enum:
writer.WriteValue((int)value);
return;
case Guid g:
writer.WriteValue(g.ToString());
return;
case Color c:
writer.WriteValue(c.ToArgb());
return;
case DateTime dt:
writer.WriteValue(dt.ToString("o", CultureInfo.InvariantCulture));
return;
case Matrix4x4 md:
writer.WriteStartArray();
writer.WriteValue(md.M11);
writer.WriteValue(md.M12);
writer.WriteValue(md.M13);
writer.WriteValue(md.M14);
writer.WriteValue(md.M21);
writer.WriteValue(md.M22);
writer.WriteValue(md.M23);
writer.WriteValue(md.M24);
writer.WriteValue(md.M31);
writer.WriteValue(md.M32);
writer.WriteValue(md.M33);
writer.WriteValue(md.M34);
writer.WriteValue(md.M41);
writer.WriteValue(md.M42);
writer.WriteValue(md.M43);
writer.WriteValue(md.M44);
writer.WriteEndArray();
return;
// Handle ObjectReference before Base (since ObjectReference extends Base)
// This prevents double-serialization and properly propagates closures
case ObjectReference objRef:
{
writer.WriteStartObject();
writer.WritePropertyName("speckle_type");
writer.WriteValue("reference");
writer.WritePropertyName("referencedId");
writer.WriteValue(objRef.referencedId);
writer.WriteEndObject();
// Propagate closure: add the referenced ID
closures[objRef.referencedId] = closures.TryGetValue(objRef.referencedId, out var existing) ? existing + 1 : 1;
// Propagate nested closures from the ObjectReference.closure dictionary
if (objRef.closure != null)
{
foreach (var kvp in objRef.closure)
{
closures[kvp.Key] = closures.TryGetValue(kvp.Key, out var existingDepth)
? existingDepth + kvp.Value
: kvp.Value;
}
}
return;
}
case Base baseObj:
{
if (isDetachable)
{
var childClosures = new Dictionary<string, int>();
var (childId, childJson) = SerializeBase(baseObj, true, childClosures, detachedObjects);
detachedObjects.Add((childId, childJson, childClosures, baseObj, baseObj.speckle_type));
writer.WriteStartObject();
writer.WritePropertyName("speckle_type");
writer.WriteValue("reference");
writer.WritePropertyName("referencedId");
writer.WriteValue(childId.Value);
writer.WriteEndObject();
closures[childId.Value] = closures.TryGetValue(childId.Value, out var existing) ? existing + 1 : 1;
foreach (var kvp in childClosures)
{
closures[kvp.Key] = closures.TryGetValue(kvp.Key, out var existingDepth)
? existingDepth + kvp.Value
: kvp.Value;
}
}
else
{
var inlineClosures = new Dictionary<string, int>();
var (_, inlineJson) = SerializeBase(baseObj, false, inlineClosures, detachedObjects);
writer.WriteRawValue(inlineJson.Value);
foreach (var kvp in inlineClosures)
{
closures[kvp.Key] = closures.TryGetValue(kvp.Key, out var existingDepth)
? existingDepth + kvp.Value
: kvp.Value;
}
}
return;
}
case IDictionary dict:
{
writer.WriteStartObject();
foreach (DictionaryEntry kvp in dict)
{
if (kvp.Key is not string key)
{
throw new ArgumentException("Dictionary keys must be strings", nameof(value));
}
writer.WritePropertyName(key);
SerializeValue(kvp.Value, writer, false, closures, detachedObjects);
}
writer.WriteEndObject();
return;
}
case ICollection collection:
{
writer.WriteStartArray();
foreach (var item in collection)
{
SerializeValue(item, writer, isDetachable, closures, detachedObjects);
}
writer.WriteEndArray();
return;
}
default:
// This case will handle primitives and `null`
// Will throw JsonWriterException if not supported
writer.WriteValue(value);
return;
}
}
private UploadItem ReferenceToUploadItem(ObjectReference existingRef)
{
var sb = Pools.StringBuilders.Get();
try
{
using var stringWriter = new StringWriter(sb);
using var jsonWriter = new JsonTextWriter(stringWriter);
jsonWriter.WriteStartObject();
jsonWriter.WritePropertyName("speckle_type");
jsonWriter.WriteValue("reference");
jsonWriter.WritePropertyName("referencedId");
jsonWriter.WriteValue(existingRef.referencedId);
jsonWriter.WritePropertyName("__closure");
if (existingRef.closure != null && existingRef.closure.Count > 0)
{
jsonWriter.WriteStartObject();
foreach (var kvp in existingRef.closure)
{
jsonWriter.WritePropertyName(kvp.Key);
jsonWriter.WriteValue(kvp.Value);
}
jsonWriter.WriteEndObject();
}
else
{
jsonWriter.WriteNull();
}
jsonWriter.WriteEndObject();
jsonWriter.Flush();
var refJson = new Json(stringWriter.ToString());
return new UploadItem(
existingRef.referencedId,
refJson,
existingRef.speckle_type,
existingRef // Pass through the original ObjectReference
);
}
finally
{
Pools.StringBuilders.Return(sb);
}
}
}
src/Speckle.Sdk/Pipelines/Send/Uploader.cs
-159
View File
@@ -1,159 +0,0 @@
using System.Net.Http.Headers;
using System.Text;
using Speckle.InterfaceGenerator;
using Speckle.Newtonsoft.Json;
using Speckle.Sdk.Credentials;
using Speckle.Sdk.Helpers;
using Speckle.Sdk.Logging;
using Speckle.Sdk.Pipelines.Progress;
namespace Speckle.Sdk.Pipelines.Send;
[GenerateAutoInterface]
public sealed class UploaderFactory(ISpeckleHttp httpClientFactory, ISdkActivityFactory activityFactory)
: IUploaderFactory
{
public Uploader CreateInstance(
string projectId,
string ingestionId,
Account account,
IProgress<StreamProgressArgs> progress,
CancellationToken cancellationToken
) => new(projectId, ingestionId, activityFactory, httpClientFactory, account, progress, cancellationToken);
}
public sealed class Uploader : IDisposable
{
private readonly string _projectId;
private readonly string _ingestionId;
private readonly CancellationToken _cancellationToken;
private readonly HttpClient _speckleClient;
private readonly HttpClient _s3Client;
private readonly ISdkActivityFactory _activity;
private readonly IProgress<StreamProgressArgs> _progress;
internal Uploader(
string projectId,
string ingestionId,
ISdkActivityFactory activity,
ISpeckleHttp httpClientFactory,
Account speckleAccount,
IProgress<StreamProgressArgs> progress,
CancellationToken cancellationToken
)
{
_projectId = projectId;
_ingestionId = ingestionId;
_activity = activity;
_cancellationToken = cancellationToken;
_progress = progress;
_speckleClient = httpClientFactory.CreateHttpClient(authorizationToken: speckleAccount.token);
_speckleClient.BaseAddress = new(new(speckleAccount.serverInfo.url), "/api/v1/");
_s3Client = httpClientFactory.CreateHttpClient();
}
public async Task Send(Stream fileStream)
{
PresignedUploadResponse presignedUploadResponse = await GetPresignedUrl().ConfigureAwait(false);
var etag = await UploadToS3(fileStream, presignedUploadResponse).ConfigureAwait(false);
await TriggerProcessing(new() { Etag = etag }).ConfigureAwait(false);
}
private async Task<PresignedUploadResponse> GetPresignedUrl()
{
using var a = _activity.Start("Get Presigned Url");
try
{
var signUri = new Uri($"projects/{_projectId}/modelingestion/{_ingestionId}/uploads/sign", UriKind.Relative);
using var signResponse = await _speckleClient.PostAsync(signUri, null, _cancellationToken).ConfigureAwait(false);
signResponse.EnsureSuccessStatusCode();
#if NET5_0_OR_GREATER
string signResponseString = await signResponse
.Content.ReadAsStringAsync(_cancellationToken)
.ConfigureAwait(false);
#else
string signResponseString = await signResponse.Content.ReadAsStringAsync().ConfigureAwait(false);
#endif
PresignedUploadResponse presignedUpload =
JsonConvert.DeserializeObject<PresignedUploadResponse>(signResponseString)
?? throw new InvalidOperationException("Failed to get presigned upload URL");
return presignedUpload;
}
catch (Exception ex)
{
a?.SetStatus(SdkActivityStatusCode.Error);
a?.RecordException(ex);
throw;
}
}
private async Task<string> UploadToS3(Stream fileStream, PresignedUploadResponse presignedUploadResponse)
{
using var a = _activity.Start("Uploading file to pre-signed url");
try
{
Stream progressStream = new ProgressStream(fileStream, _progress);
using var streamContent = new StreamContent(progressStream);
streamContent.Headers.ContentType = new MediaTypeHeaderValue("application/octet-stream");
streamContent.Headers.ContentLength = fileStream.Length;
using var uploadRequest = new HttpRequestMessage(HttpMethod.Put, presignedUploadResponse.Url);
foreach (var kvp in presignedUploadResponse.AdditionalRequestHeaders)
{
uploadRequest.Headers.Add(kvp.Key, kvp.Value);
}
uploadRequest.Content = streamContent;
using var uploadResponse = await _s3Client
.SendAsync(uploadRequest, HttpCompletionOption.ResponseHeadersRead, _cancellationToken)
.ConfigureAwait(false);
uploadResponse.EnsureSuccessStatusCode();
return BlobApiHelpers.ParseEtagHeader(uploadResponse.Headers);
}
catch (Exception ex)
{
a?.SetStatus(SdkActivityStatusCode.Error);
a?.RecordException(ex);
throw;
}
}
private async Task TriggerProcessing(TriggerUploadRequest request)
{
using var a = _activity.Start("Triggering Processing");
try
{
Uri processUri = new($"projects/{_projectId}/modelingestion/{_ingestionId}/uploads/process", UriKind.Relative);
string requestBody = JsonConvert.SerializeObject(request);
using var content = new StringContent(requestBody, Encoding.UTF8, "application/json");
using HttpResponseMessage processResponse = await _speckleClient
.PostAsync(processUri, content, _cancellationToken)
.ConfigureAwait(false);
string body = await processResponse.Content.ReadAsStringAsync().ConfigureAwait(false);
processResponse.EnsureSuccessStatusCode();
}
catch (Exception ex)
{
a?.SetStatus(SdkActivityStatusCode.Error);
a?.RecordException(ex);
throw;
}
}
public void Dispose()
{
_speckleClient.Dispose();
_s3Client.Dispose();
}
}
src/Speckle.Sdk/Pipelines/Send/UploaderDTOs.cs
-20
View File
@@ -1,20 +0,0 @@
using Speckle.Newtonsoft.Json;
using Speckle.Sdk.Models;
using Speckle.Sdk.Serialisation;
namespace Speckle.Sdk.Pipelines.Send;
public record UploadItem(string Id, Json Json, string SpeckleType, ObjectReference Reference);
internal record PresignedUploadResponse
{
public required Uri Url { get; init; }
public required string Key { get; init; }
public Dictionary<string, string> AdditionalRequestHeaders { get; init; } = new();
}
internal readonly struct TriggerUploadRequest
{
[JsonProperty("etag")]
public required string Etag { get; init; }
}
src/Speckle.Sdk/Serialisation/V2/MemoryServerObjectManager.cs
-1
View File
@@ -9,7 +9,6 @@ public class MemoryServerObjectManager(ConcurrentDictionary<string, string> obje
{
public virtual async IAsyncEnumerable<(string, string)> DownloadObjects(
IReadOnlyCollection<string> objectIds,
string? attributeMask,
IProgress<ProgressArgs>? progress,
[EnumeratorCancellation] CancellationToken cancellationToken
)
src/Speckle.Sdk/Serialisation/V2/Receive/DeserializeProcess.cs
+1 -1
View File
@@ -10,7 +10,7 @@ using Speckle.Sdk.Transports;
namespace Speckle.Sdk.Serialisation.V2.Receive;
public record DeserializeProcessOptions(
bool SkipCache = false, //TODO: This appears to be bugged when set to `true`, `LoadId` depends on sqlite
bool SkipCache = false,
bool ThrowOnMissingReferences = true,
bool SkipInvalidConverts = false,
int? MaxParallelism = null,
src/Speckle.Sdk/Serialisation/V2/Receive/ObjectLoader.cs
+9 -9
View File
@@ -19,7 +19,9 @@ public sealed class ObjectLoader(
IProgress<ProgressArgs>? progress,
ILogger<ObjectLoader> logger,
CancellationToken cancellationToken
#pragma warning disable CS9107 // Parameter is captured into the state of the enclosing type and its value is also passed to the base constructor. The value might be captured by the base class as well.
) : ChannelLoader<BaseItem>(cancellationToken), IObjectLoader
#pragma warning restore CS9107 // Parameter is captured into the state of the enclosing type and its value is also passed to the base constructor. The value might be captured by the base class as well.
{
private int? _allChildrenCount;
private long _checkCache;
@@ -27,7 +29,6 @@ public sealed class ObjectLoader(
private long _downloaded;
private long _totalToDownload;
private DeserializeProcessOptions _options = new();
private readonly CancellationToken _cancellationToken = cancellationToken;
[AutoInterfaceIgnore]
public void Dispose() => sqLiteJsonCacheManager.Dispose();
@@ -45,7 +46,7 @@ public sealed class ObjectLoader(
{
//assume everything exists as the root is there.
var allChildren = ClosureParser
.GetClosuresSorted(rootJson, _cancellationToken)
.GetClosuresSorted(rootJson, cancellationToken)
.Select(x => new Id(x.Item1))
.ToList();
//this probably yields away from the Main thread to let host apps update progress
@@ -58,11 +59,11 @@ public sealed class ObjectLoader(
if (!options.SkipServer)
{
rootJson = await serverObjectManager
.DownloadSingleObject(rootId, progress, _cancellationToken)
.DownloadSingleObject(rootId, progress, cancellationToken)
.NotNull()
.ConfigureAwait(false);
IReadOnlyCollection<Id> allChildrenIds = ClosureParser
.GetClosures(rootJson, _cancellationToken)
.GetClosures(rootJson, cancellationToken)
.OrderByDescending(x => x.Item2)
.Select(x => new Id(x.Item1))
.Where(x => !x.Value.StartsWith("blob", StringComparison.Ordinal))
@@ -110,13 +111,12 @@ public sealed class ObjectLoader(
await foreach (
var (id, json) in serverObjectManager.DownloadObjects(
ids.Select(x => x.NotNull()).ToList(),
null, //TODO: Implement attribute masking in a safe way that will not poison SQLite DB.
progress,
_cancellationToken
cancellationToken
)
)
{
_cancellationToken.ThrowIfCancellationRequested();
cancellationToken.ThrowIfCancellationRequested();
Interlocked.Increment(ref _downloaded);
progress?.Report(new(ProgressEvent.DownloadObjects, _downloaded, _totalToDownload));
toCache.Add(new(new(id), new(json), true, null));
@@ -138,7 +138,7 @@ public sealed class ObjectLoader(
{
if (!_options.SkipCache)
{
_cancellationToken.ThrowIfCancellationRequested();
cancellationToken.ThrowIfCancellationRequested();
sqLiteJsonCacheManager.SaveObjects(batch.Select(x => (x.Id.Value, x.Json.Value)));
Interlocked.Exchange(ref _cached, _cached + batch.Count);
progress?.Report(new(ProgressEvent.CachedToLocal, _cached, _allChildrenCount));
@@ -168,7 +168,7 @@ public sealed class ObjectLoader(
private void ThrowIfFailed()
{
//always check for cancellation first
_cancellationToken.ThrowIfCancellationRequested();
cancellationToken.ThrowIfCancellationRequested();
if (Exception is not null)
{
throw new SpeckleException($"Error while loading: {Exception.Message}", Exception);
src/Speckle.Sdk/Serialisation/V2/Send/BaseItem.cs
+1 -7
View File
@@ -2,13 +2,7 @@ using System.Text;
namespace Speckle.Sdk.Serialisation.V2.Send;
public sealed record BaseItem(
Id Id,
Json Json,
bool NeedsStorage,
Dictionary<Id, int>? Closures,
bool? IsReference = false
) : IHasByteSize
public sealed record BaseItem(Id Id, Json Json, bool NeedsStorage, Dictionary<Id, int>? Closures) : IHasByteSize
{
public int ByteSize { get; } = Encoding.UTF8.GetByteCount(Json.Value);
src/Speckle.Sdk/Serialisation/V2/Send/BaseSerializer.cs
+7 -2
View File
@@ -21,7 +21,12 @@ public class BaseSerializer(
public IReadOnlyDictionary<Id, ObjectReference> ObjectReferences => _objectReferences;
//leave this sync
public IEnumerable<BaseItem> Serialise(Base obj, bool skipCacheRead, CancellationToken cancellationToken)
public IEnumerable<BaseItem> Serialise(
Base obj,
IReadOnlyDictionary<Id, NodeInfo> childInfo,
bool skipCacheRead,
CancellationToken cancellationToken
)
{
if (!skipCacheRead && obj.id != null)
{
@@ -33,7 +38,7 @@ public class BaseSerializer(
}
}
using var serializer2 = objectSerializerFactory.Create(cancellationToken);
using var serializer2 = objectSerializerFactory.Create(childInfo, cancellationToken);
var items = _pool.Get();
try
{
src/Speckle.Sdk/Serialisation/V2/Send/ObjectSaver.cs
+2 -2
View File
@@ -20,10 +20,10 @@ public sealed class ObjectSaver(
ISqLiteJsonCacheManager sqLiteJsonCacheManager,
IServerObjectManager serverObjectManager,
ILogger<ObjectSaver> logger,
SerializeProcessOptions options,
CancellationToken cancellationToken
CancellationToken cancellationToken,
#pragma warning disable CS9107
#pragma warning disable CA2254
SerializeProcessOptions? options = null
) : ChannelSaver<BaseItem>, IObjectSaver
#pragma warning restore CA2254
#pragma warning restore CS9107
src/Speckle.Sdk/Serialisation/V2/Send/ObjectSerializer.cs
+26 -8
View File
@@ -26,6 +26,8 @@ public sealed class ObjectSerializer : IObjectSerializer
{
private HashSet<object> _parentObjects = new();
private readonly IReadOnlyDictionary<Id, NodeInfo> _childCache;
private readonly IBasePropertyGatherer _propertyGatherer;
private readonly CancellationToken _cancellationToken;
@@ -50,6 +52,7 @@ public sealed class ObjectSerializer : IObjectSerializer
/// <param name="cancellationToken"></param>
public ObjectSerializer(
IBasePropertyGatherer propertyGatherer,
IReadOnlyDictionary<Id, NodeInfo> childCache,
Pool<List<(Id, Json, Closures)>> chunksPool,
Pool<List<DataChunk>> chunks2Pool,
Pool<List<object?>> chunks3Pool,
@@ -57,6 +60,7 @@ public sealed class ObjectSerializer : IObjectSerializer
)
{
_propertyGatherer = propertyGatherer;
_childCache = childCache;
_chunksPool = chunksPool;
_chunks2Pool = chunks2Pool;
_chunks3Pool = chunks3Pool;
@@ -295,14 +299,28 @@ public sealed class ObjectSerializer : IObjectSerializer
private (Id, Json)? SerializeDetachedBase(Base baseObj, Closures closures)
{
Closures childClosures = [];
var sb = Pools.StringBuilders.Get();
using var writer = new StringWriter(sb);
using var jsonWriter = SpeckleObjectSerializerPool.Instance.GetJsonTextWriter(writer);
var id = SerializeBaseWithClosures(baseObj, jsonWriter, childClosures, true);
var json = new Json(writer.ToString());
Pools.StringBuilders.Return(sb);
closures.IncrementClosures(childClosures);
Closures childClosures;
Id id;
Json json;
//avoid multiple serialization to get closures
if (baseObj.id != null && _childCache.TryGetValue(new(baseObj.id), out var info))
{
id = new Id(baseObj.id);
childClosures = info.GetClosures(_cancellationToken);
json = info.Json;
closures.IncrementClosures(childClosures);
}
else
{
childClosures = [];
var sb = Pools.StringBuilders.Get();
using var writer = new StringWriter(sb);
using var jsonWriter = SpeckleObjectSerializerPool.Instance.GetJsonTextWriter(writer);
id = SerializeBaseWithClosures(baseObj, jsonWriter, childClosures, true);
closures.IncrementClosures(childClosures);
json = new Json(writer.ToString());
Pools.StringBuilders.Return(sb);
}
var json2 = ReferenceGenerator.CreateReference(id);
closures.MergeClosure(id);
// add to obj refs to return
src/Speckle.Sdk/Serialisation/V2/Send/ObjectSerializerFactory.cs
+2 -2
View File
@@ -12,6 +12,6 @@ public class ObjectSerializerFactory(IBasePropertyGatherer propertyGatherer) : I
private readonly Pool<List<DataChunk>> _chunk2Pool = Pools.CreateListPool<DataChunk>();
private readonly Pool<List<object?>> _chunk3Pool = Pools.CreateListPool<object?>();
public IObjectSerializer Create(CancellationToken cancellationToken) =>
new ObjectSerializer(propertyGatherer, _chunkPool, _chunk2Pool, _chunk3Pool, cancellationToken);
public IObjectSerializer Create(IReadOnlyDictionary<Id, NodeInfo> baseCache, CancellationToken cancellationToken) =>
new ObjectSerializer(propertyGatherer, baseCache, _chunkPool, _chunk2Pool, _chunk3Pool, cancellationToken);
}
src/Speckle.Sdk/Serialisation/V2/Send/SerializeProcess.cs
+45 -22
View File
@@ -1,3 +1,4 @@
using System.Collections.Concurrent;
using System.Diagnostics.CodeAnalysis;
using Microsoft.Extensions.Logging;
using Speckle.InterfaceGenerator;
@@ -36,8 +37,8 @@ public sealed class SerializeProcess(
IBaseChildFinder baseChildFinder,
IBaseSerializer baseSerializer,
ILoggerFactory loggerFactory,
SerializeProcessOptions options,
CancellationToken cancellationToken
CancellationToken cancellationToken,
SerializeProcessOptions? options = null
) : ISerializeProcess
{
private static readonly Dictionary<Id, NodeInfo> EMPTY_CLOSURES = new();
@@ -63,8 +64,13 @@ public sealed class SerializeProcess(
ThreadPriority.BelowNormal,
Environment.ProcessorCount * 2
);
private readonly SerializeProcessOptions _options = options ?? new();
private readonly Pool<Dictionary<Id, NodeInfo>> _currentClosurePool = Pools.CreateDictionaryPool<Id, NodeInfo>();
private readonly Pool<ConcurrentDictionary<Id, NodeInfo>> _childClosurePool = Pools.CreateConcurrentDictionaryPool<
Id,
NodeInfo
>();
private readonly Pool<List<Task<Dictionary<Id, NodeInfo>>>> _taskResultPool = Pools.CreateListPool<
Task<Dictionary<Id, NodeInfo>>
@@ -107,12 +113,22 @@ public sealed class SerializeProcess(
try
{
var channelTask = objectSaver.Start(
options.MaxParallelism,
options.MaxHttpSendBatchSize,
options.MaxCacheBatchSize,
options?.MaxParallelism,
options?.MaxHttpSendBatchSize,
options?.MaxCacheBatchSize,
_processSource.Token
);
var findTotalObjectsTask = Task.CompletedTask;
if (!_options.SkipFindTotalObjects)
{
ThrowIfFailed();
findTotalObjectsTask = Task.Factory.StartNew(
() => TraverseTotal(root),
_processSource.Token,
TaskCreationOptions.AttachedToParent | TaskCreationOptions.PreferFairness,
_highest
);
}
await Traverse(root).ConfigureAwait(false);
ThrowIfFailed();
@@ -123,7 +139,6 @@ public sealed class SerializeProcess(
ThrowIfFailed();
await WaitForSchedulerCompletion().ConfigureAwait(false);
ThrowIfFailed();
return new(root.id.NotNull(), baseSerializer.ObjectReferences.Freeze());
}
catch (OperationCanceledException)
@@ -210,6 +225,7 @@ public sealed class SerializeProcess(
return EMPTY_CLOSURES;
}
var childClosures = _childClosurePool.Get();
foreach (var childClosure in taskClosures)
{
if (IsCancelled())
@@ -218,6 +234,7 @@ public sealed class SerializeProcess(
}
foreach (var kvp in childClosure)
{
childClosures[kvp.Key] = kvp.Value;
if (IsCancelled())
{
return EMPTY_CLOSURES;
@@ -232,7 +249,7 @@ public sealed class SerializeProcess(
return EMPTY_CLOSURES;
}
var items = baseSerializer.Serialise(obj, options.SkipCacheRead, _processSource.Token);
var items = baseSerializer.Serialise(obj, childClosures, _options.SkipCacheRead, _processSource.Token);
if (IsCancelled())
{
@@ -240,27 +257,33 @@ public sealed class SerializeProcess(
}
var currentClosures = _currentClosurePool.Get();
Interlocked.Increment(ref _objectCount);
progress?.Report(new(ProgressEvent.FromCacheOrSerialized, _objectCount, Math.Max(_objectCount, _objectsFound)));
foreach (var item in items)
try
{
if (IsCancelled())
Interlocked.Increment(ref _objectCount);
progress?.Report(new(ProgressEvent.FromCacheOrSerialized, _objectCount, Math.Max(_objectCount, _objectsFound)));
foreach (var item in items)
{
return EMPTY_CLOSURES;
}
if (IsCancelled())
{
return EMPTY_CLOSURES;
}
if (item.NeedsStorage)
{
Interlocked.Increment(ref _objectsSerialized);
await objectSaver.SaveAsync(item).ConfigureAwait(false);
}
if (item.NeedsStorage)
{
Interlocked.Increment(ref _objectsSerialized);
await objectSaver.SaveAsync(item).ConfigureAwait(false);
}
if (!currentClosures.ContainsKey(item.Id))
{
currentClosures.Add(item.Id, new NodeInfo(item.Json, item.Closures));
if (!currentClosures.ContainsKey(item.Id))
{
currentClosures.Add(item.Id, new NodeInfo(item.Json, item.Closures));
}
}
}
finally
{
_childClosurePool.Return(childClosures);
}
return currentClosures;
}
src/Speckle.Sdk/Serialisation/V2/SerializeProcessFactory.cs
+2 -3
View File
@@ -44,14 +44,13 @@ public class SerializeProcessFactory(
sqLiteJsonCacheManager,
serverObjectManager,
loggerFactory.CreateLogger<ObjectSaver>(),
options ?? new SerializeProcessOptions(),
cancellationToken
),
baseChildFinder,
new BaseSerializer(sqLiteJsonCacheManager, objectSerializerFactory),
loggerFactory,
options ?? new SerializeProcessOptions(),
cancellationToken
cancellationToken,
options
);
public ISerializeProcess CreateSerializeProcess(
src/Speckle.Sdk/Serialisation/V2/ServerObjectManager.cs
+2 -7
View File
@@ -51,7 +51,6 @@ public class ServerObjectManager : IServerObjectManager
public async IAsyncEnumerable<(string, string)> DownloadObjects(
IReadOnlyCollection<string> objectIds,
string? attributeMask,
IProgress<ProgressArgs>? progress,
[EnumeratorCancellation] CancellationToken cancellationToken
)
@@ -60,14 +59,10 @@ public class ServerObjectManager : IServerObjectManager
cancellationToken.ThrowIfCancellationRequested();
using var childrenHttpMessage = new HttpRequestMessage();
childrenHttpMessage.RequestUri = new Uri($"/api/v2/projects/{_streamId}/object-stream/", UriKind.Relative);
childrenHttpMessage.RequestUri = new Uri($"/api/getobjects/{_streamId}", UriKind.Relative);
childrenHttpMessage.Method = HttpMethod.Post;
Dictionary<string, object> postParameters = new() { { "objectIds", objectIds } };
if (!string.IsNullOrWhiteSpace(attributeMask))
{
postParameters.Add("attributeMask", attributeMask.NotNull());
}
Dictionary<string, string> postParameters = new() { { "objects", JsonConvert.SerializeObject(objectIds) } };
string serializedPayload = JsonConvert.SerializeObject(postParameters);
childrenHttpMessage.Content = new StringContent(serializedPayload, Encoding.UTF8, "application/json");
childrenHttpMessage.Headers.Add("Accept", "text/plain");

Some files were not shown because too many files have changed in this diff Show More