huanld/speckle-server
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

Merge branch 'main' of github.com:specklesystems/speckle-server into gergo/adminFacelift

Browse Source
This commit is contained in:
Gergő Jedlicska
2023-07-31 16:25:35 +02:00
parent 56555342e4 3e5b5b9e30
commit 48588e5ee5
50 changed files with 3668 additions and 285 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.circleci/config.yml
+6 -6
View File
@@ -322,7 +322,7 @@ jobs:
test-server:
docker:
- image: cimg/node:18.16.1
- image: cimg/node:18.17.0
- image: cimg/redis:7.0.7
- image: 'cimg/postgres:14.5'
environment:
@@ -417,7 +417,7 @@ jobs:
test-frontend-2:
docker:
- image: cimg/node:18.16.1-browsers
- image: cimg/node:18.17.0-browsers
resource_class: xlarge
steps:
- checkout
@@ -460,7 +460,7 @@ jobs:
test-dui-3:
docker:
- image: cimg/node:18.16.1
- image: cimg/node:18.17.0
resource_class: medium+
steps:
- checkout
@@ -490,7 +490,7 @@ jobs:
test-ui-components:
docker:
- image: cimg/node:18.16.1-browsers
- image: cimg/node:18.17.0-browsers
resource_class: xlarge
steps:
- checkout
@@ -544,7 +544,7 @@ jobs:
frontend-2-chromatic:
resource_class: medium+
docker:
- image: cimg/node:18.16.1
- image: cimg/node:18.17.0
steps:
- checkout
- restore_cache:
@@ -578,7 +578,7 @@ jobs:
ui-components-chromatic:
resource_class: medium+
docker:
- image: cimg/node:18.16.1
- image: cimg/node:18.17.0
steps:
- checkout
- restore_cache:
package.json
+1 -1
View File
@@ -6,7 +6,7 @@
"name": "root",
"private": true,
"engines": {
"node": "^18.16.1"
"node": "^18.17.0"
},
"scripts": {
"build": "yarn workspaces foreach -ptv run build",
packages/fileimport-service/Dockerfile
+2 -2
View File
@@ -1,6 +1,6 @@
ARG NODE_ENV=production
FROM node:18.16.1-bullseye-slim as build-stage
FROM node:18.17.0-bullseye-slim as build-stage
ARG NODE_ENV
ENV NODE_ENV=${NODE_ENV}
@@ -47,7 +47,7 @@ RUN apt-get update && \
COPY packages/fileimport-service/requirements.txt /speckle-server/
RUN /venv/bin/pip install --disable-pip-version-check --no-cache-dir --requirement /speckle-server/requirements.txt
FROM node:18.16.1-bullseye-slim as dependency-stage
FROM node:18.17.0-bullseye-slim as dependency-stage
# installing just the production dependencies
# separate stage to avoid including development dependencies
ARG NODE_ENV
packages/fileimport-service/package.json
+1 -1
View File
@@ -12,7 +12,7 @@
"url": "git+https://github.com/specklesystems/speckle-server.git"
},
"engines": {
"node": "^18.16.1"
"node": "^18.17.0"
},
"scripts": {
"dev": "cross-env POSTGRES_URL=postgres://speckle:speckle@localhost/speckle NODE_ENV=development LOG_PRETTY=true SPECKLE_SERVER_URL=http://localhost:3000 nodemon --no-experimental-fetch ./src/daemon.js",
packages/frontend/Dockerfile
+1 -1
View File
@@ -2,7 +2,7 @@
ARG NODE_ENV=production
ARG SPECKLE_SERVER_VERSION=custom
# build stage
FROM node:18.16.1-bullseye-slim as build-stage
FROM node:18.17.0-bullseye-slim as build-stage
ARG NODE_ENV
ARG SPECKLE_SERVER_VERSION
packages/frontend/package.json
+1 -1
View File
@@ -97,6 +97,6 @@
"vue-tsc": "^1.0.8"
},
"engines": {
"node": "^18.16.1"
"node": "^18.17.0"
}
}
packages/objectloader/package.json
+1 -1
View File
@@ -12,7 +12,7 @@
"directory": "packages/objectloader"
},
"engines": {
"node": "^18.16.1"
"node": "^18.17.0"
},
"scripts": {
"lint": "eslint . --ext .js,.ts",
packages/preview-service/Dockerfile
+2 -2
View File
@@ -1,7 +1,7 @@
# NOTE: Docker context should be set to git root directory, to include the viewer
ARG NODE_ENV=production
FROM node:18.16.1-bullseye-slim as build-stage
FROM node:18.17.0-bullseye-slim as build-stage
ARG NODE_ENV
ENV NODE_ENV=${NODE_ENV}
@@ -36,7 +36,7 @@ COPY packages/preview-service ./packages/preview-service/
# This way the foreach only builds the frontend and its deps
RUN yarn workspaces foreach run build
FROM node:18.16.1-bullseye-slim as node
FROM node:18.17.0-bullseye-slim as node
RUN apt-get update && \
DEBIAN_FRONTEND=noninteractive apt-get install -y \
packages/preview-service/package.json
+1 -1
View File
@@ -11,7 +11,7 @@
"directory": "packages/preview-service"
},
"engines": {
"node": "^18.16.1"
"node": "^18.17.0"
},
"scripts": {
"dev": "LOG_PRETTY=true nodemon --trace-deprecation ./bin/www",
packages/server/.env-example
+5 -3
View File
@@ -22,9 +22,11 @@ REDIS_URL="redis://127.0.0.1:6379"
USE_FRONTEND_2=false
FRONTEND_ORIGIN="http://127.0.0.1:8081"
# Stream to be used as the demo/tutorial stream in onboarding flows
# (if not set/valid, stream will be a blank one)
ONBOARDING_STREAM_ID=
# URL of a project on any FE2 speckle server that will be pulled in and used as the onboarding stream
ONBOARDING_STREAM_URL=https://latest.speckle.systems/projects/843d07eb10
# Increase this value to re-sync the onboarding stream
ONBOARDING_STREAM_CACHE_BUST_NUMBER=1
############################################################
# Postgres Database
packages/server/Dockerfile
+3 -3
View File
@@ -1,7 +1,7 @@
ARG NODE_ENV=production
ARG SPECKLE_SERVER_VERSION=custom
FROM node:18.16.1-bullseye-slim as build-stage
FROM node:18.17.0-bullseye-slim as build-stage
ARG NODE_ENV
ARG SPECKLE_SERVER_VERSION
WORKDIR /speckle-server
@@ -39,7 +39,7 @@ RUN yarn workspaces foreach run build
# install only production dependencies
# we need a clean environment, free of build dependencies
FROM node:18.16.1-bullseye-slim as dependency-stage
FROM node:18.17.0-bullseye-slim as dependency-stage
ARG NODE_ENV
ARG SPECKLE_SERVER_VERSION
@@ -56,7 +56,7 @@ COPY packages/objectloader/package.json ./packages/objectloader/
WORKDIR /speckle-server/packages/server
RUN yarn workspaces focus --production
FROM node:18.16.1-bullseye-slim as production-stage
FROM node:18.17.0-bullseye-slim as production-stage
ARG NODE_ENV
ARG SPECKLE_SERVER_VERSION
ARG FILE_SIZE_LIMIT_MB=100
packages/server/codegen.yml
+10
View File
@@ -30,6 +30,16 @@ generates:
Comment: '@/modules/comments/helpers/graphTypes#CommentGraphQLReturn'
PendingStreamCollaborator: '@/modules/serverinvites/helpers/graphTypes#PendingStreamCollaboratorGraphQLReturn'
FileUpload: '@/modules/fileuploads/helpers/types#FileUploadGraphQLReturn'
modules/cross-server-sync/graph/generated/graphql.ts:
plugins:
- 'typescript'
- 'typescript-operations'
documents:
- 'modules/cross-server-sync/**/*.{js,ts}'
config:
scalars:
JSONObject: Record<string, unknown>
DateTime: string
test/graphql/generated/graphql.ts:
plugins:
- 'typescript'
packages/server/logging/logging.ts
+3
View File
@@ -28,4 +28,7 @@ export const dbNotificationLogger = extendLoggerComponent(logger, 'db-notificati
export const mixpanelLogger = extendLoggerComponent(logger, 'mixpanel')
export const graphqlLogger = extendLoggerComponent(logger, 'graphql')
export const authLogger = extendLoggerComponent(logger, 'auth')
export const crossServerSyncLogger = extendLoggerComponent(logger, 'cross-server-sync')
export type Logger = typeof logger
export { extendLoggerComponent }
packages/server/modules/cli/commands/download/commit.ts
+3 -2
View File
@@ -1,5 +1,6 @@
import { CommandModule } from 'yargs'
import { downloadCommit } from '@/modules/cli/services/download/commit'
import { downloadCommit } from '@/modules/cross-server-sync/services/commit'
import { cliLogger } from '@/logging/logging'
const command: CommandModule<
unknown,
@@ -41,7 +42,7 @@ const command: CommandModule<
}
},
handler: async (argv) => {
await downloadCommit(argv)
await downloadCommit(argv, { logger: cliLogger })
}
}
packages/server/modules/cli/commands/download/project.ts
+36
View File
@@ -0,0 +1,36 @@
import { CommandModule } from 'yargs'
import { cliLogger } from '@/logging/logging'
import { downloadProject } from '@/modules/cross-server-sync/services/project'
const command: CommandModule<
unknown,
{
projectUrl: string
authorId: string
syncComments: boolean
}
> = {
command: 'project <projectUrl> <authorId> [syncComments]',
describe: 'Download a project from an external Speckle server instance',
builder: {
projectUrl: {
describe:
'Public Project URL (e.g. https://latest.speckle.systems/projects/594d657cdd)',
type: 'string'
},
authorId: {
describe: 'ID of the local user that will own the project',
type: 'string'
},
syncComments: {
describe: 'Whether or not to sync comments as well',
type: 'boolean',
default: true
}
},
handler: async (argv) => {
await downloadProject(argv, { logger: cliLogger })
}
}
export = command
packages/server/modules/comments/services/commentTextService.ts
+7 -1
View File
@@ -37,7 +37,13 @@ export function buildCommentTextFromInput({
}>) {
if ((!isTextEditorDoc(doc) || isDocEmpty(doc)) && !blobIds.length) {
throw new RichTextParseError(
'Attempting to build comment text without document & attachments!'
'Attempting to build comment text without document & attachments!',
{
info: {
doc,
blobIds
}
}
)
}
packages/server/modules/core/dbSchema.ts
+4 -4
View File
@@ -66,7 +66,7 @@ type MetaInnerSchemaConfig<
* Get meta keys individually
*/
metaKey: {
[keyName in MK]: string
[keyName in MK]: keyName
}
/**
@@ -186,8 +186,8 @@ function buildMetaTableHelper<T extends string, C extends string, MK extends str
prev[curr] = curr
return prev
},
{} as Record<keyof BaseMetaRecord | MK, string>
),
{} as Record<keyof BaseMetaRecord | MK, keyof BaseMetaRecord | MK>
) as { [keyName in MK]: keyName },
parentIdentityCol
})
@@ -218,7 +218,7 @@ function buildMetaTableHelper<T extends string, C extends string, MK extends str
export const StreamsMeta = buildMetaTableHelper(
'streams_meta',
['streamId', 'key', 'value', 'createdAt', 'updatedAt'],
['viewerE2eTestStreamVersion'],
['onboardingBaseStream'],
'streamId'
)
packages/server/modules/core/repositories/streams.ts
+6 -6
View File
@@ -964,26 +964,26 @@ export async function revokeStreamPermissions(params: {
}
/**
* Marking stream as being used for a specific version of viewer e2e tests
* Mark stream as the onboarding base stream from which user onboarding streams will be cloned
*/
export async function markStreamViewerE2eTest(streamId: string, version: string) {
export async function markOnboardingBaseStream(streamId: string, version: string) {
const stream = await getStream({ streamId })
if (!stream) {
throw new Error(`Stream ${streamId} not found`)
}
const meta = metaHelpers(Streams)
await meta.set(streamId, 'viewerE2eTestStreamVersion', version)
await meta.set(streamId, Streams.meta.metaKey.onboardingBaseStream, version)
}
/**
* Get stream used for a specific version of viewer e2e tests
* Get onboarding base stream, if any
*/
export async function getViewerE2eTestStream(version: string) {
export async function getOnboardingBaseStream(version: string) {
const q = Streams.knex()
.select<StreamRecord[]>(Streams.cols)
.innerJoin(Streams.meta.name, Streams.meta.col.streamId, Streams.col.id)
.where(Streams.meta.col.key, 'viewerE2eTestStreamVersion')
.where(Streams.meta.col.key, Streams.meta.metaKey.onboardingBaseStream)
.andWhereRaw(`${Streams.meta.col.value}::text = ?`, JSON.stringify(version))
.first()
packages/server/modules/core/repositories/users.ts
+10 -1
View File
@@ -4,8 +4,8 @@ import { Nullable } from '@/modules/shared/helpers/typeHelper'
import { clamp, isArray } from 'lodash'
import { metaHelpers } from '@/modules/core/helpers/meta'
import { UserValidationError } from '@/modules/core/errors/user'
import { ServerRoles } from '@speckle/shared'
import { Knex } from 'knex'
import { Roles, ServerRoles } from '@speckle/shared'
export type UserWithOptionalRole<User extends LimitedUserRecord = UserRecord> = User & {
/**
@@ -196,3 +196,12 @@ export async function updateUser(
const [newUser] = await Users.knex().where(Users.col.id, userId).update(update, '*')
return newUser as Nullable<UserRecord>
}
export async function getFirstAdmin() {
const q = Users.knex()
.select<UserRecord[]>(Users.cols)
.innerJoin(ServerAcl.name, ServerAcl.col.userId, Users.col.id)
.where(ServerAcl.col.role, Roles.Server.Admin)
return await q.first()
}
packages/server/modules/core/rest/diffUpload.js
+15 -4
View File
@@ -6,6 +6,8 @@ const { validatePermissionsWriteStream } = require('./authUtils')
const { hasObjects } = require('../services/objects')
const { chunk } = require('lodash')
module.exports = (app) => {
app.options('/api/diff/:streamId', corsMiddleware())
@@ -26,10 +28,19 @@ module.exports = (app) => {
req.log.info(`Diffing ${objectList.length} objects.`)
const response = await hasObjects({
streamId: req.params.streamId,
objectIds: objectList
})
const chunkSize = 1000
const objectListChunks = chunk(objectList, chunkSize)
const mappedObjects = await Promise.all(
objectListChunks.map((objectListChunk) =>
hasObjects({
streamId: req.params.streamId,
objectIds: objectListChunk
})
)
)
const response = {}
Object.assign(response, ...mappedObjects)
req.log.debug(response)
res.writeHead(200, {
'Content-Encoding': 'gzip',
packages/server/modules/core/services/streams/onboarding.ts
+15 -21
View File
@@ -1,34 +1,28 @@
import { Nullable, Optional } from '@speckle/shared'
import { getOnboardingStreamId } from '@/modules/shared/helpers/envHelper'
import { Optional } from '@speckle/shared'
import { StreamCloneError } from '@/modules/core/errors/stream'
import { cloneStream } from '@/modules/core/services/streams/clone'
import { StreamRecord } from '@/modules/core/helpers/types'
import { logger } from '@/logging/logging'
import { createStreamReturnRecord } from '@/modules/core/services/streams/management'
async function cloneOnboardingStream(userId: string, sourceStreamId: Nullable<string>) {
if (!sourceStreamId) {
throw new StreamCloneError('Onboarding stream ID undefined, check env vars')
}
return await cloneStream(userId, sourceStreamId)
}
import { getOnboardingBaseProject } from '@/modules/cross-server-sync/services/onboardingProject'
export async function createOnboardingStream(targetUserId: string) {
const sourceStreamId = getOnboardingStreamId()
const sourceStream = await getOnboardingBaseProject()
let newStream: Optional<StreamRecord> = undefined
try {
newStream = await cloneOnboardingStream(targetUserId, sourceStreamId)
} catch (e) {
if (!(e instanceof StreamCloneError)) {
throw e
} else {
logger.warn(e, 'Stream clone failed')
if (sourceStream) {
let newStream: Optional<StreamRecord> = undefined
try {
newStream = await cloneStream(targetUserId, sourceStream.id)
} catch (e) {
if (!(e instanceof StreamCloneError)) {
throw e
} else {
logger.warn(e, 'Stream clone failed')
}
}
}
if (newStream) return newStream
if (newStream) return newStream
}
// clone failed, just create empty stream
return await createStreamReturnRecord({ ownerId: targetUserId })
packages/server/modules/cross-server-sync/errors/index.ts
+11
View File
@@ -0,0 +1,11 @@
import { BaseError } from '@/modules/shared/errors'
export class CrossServerCommitSyncError extends BaseError {
static code = 'CROSS_SERVER_COMMIT_SYNC_ERROR'
static defaultMessage = 'Cross-server commit sync failed unexpectedly'
}
export class CrossServerProjectSyncError extends BaseError {
static code = 'CROSS_SERVER_PROJECT_SYNC_ERROR'
static defaultMessage = 'Cross-server project sync failed unexpectedly'
}
packages/server/modules/cross-server-sync/graph/generated/graphql.ts
+2689
View File
File diff suppressed because it is too large Load Diff
packages/server/modules/cross-server-sync/index.ts
+17
View File
@@ -0,0 +1,17 @@
import { moduleLogger, crossServerSyncLogger } from '@/logging/logging'
import { ensureOnboardingProject } from '@/modules/cross-server-sync/services/onboardingProject'
import { SpeckleModule } from '@/modules/shared/helpers/typeHelper'
const crossServerSyncModule: SpeckleModule = {
init() {
moduleLogger.info('🔄️ Init cross-server-sync module')
},
finalize() {
crossServerSyncLogger.info('⬇️ Ensuring base onboarding stream asynchronously...')
void ensureOnboardingProject().catch((err) =>
crossServerSyncLogger.error(err, 'Error ensuring onboarding stream')
)
}
}
export = crossServerSyncModule
packages/server/modules/cli/services/download/commit.ts → packages/server/modules/cross-server-sync/services/commit.ts
+152 -155
View File
@@ -1,20 +1,7 @@
import fetch from 'cross-fetch'
import {
ApolloClient,
InMemoryCache,
NormalizedCacheObject,
gql,
HttpLink,
ApolloQueryResult
} from '@apollo/client/core'
import { setContext } from '@apollo/client/link/context'
import { getFrontendOrigin, getServerVersion } from '@/modules/shared/helpers/envHelper'
import {
Commit,
ViewerResourceGroup,
Comment,
CreateCommentInput
} from '@/test/graphql/generated/graphql'
import { ApolloClient, NormalizedCacheObject, gql } from '@apollo/client/core'
import { getFrontendOrigin } from '@/modules/shared/helpers/envHelper'
import { CreateCommentInput } from '@/test/graphql/generated/graphql'
import { getStreamBranchByName } from '@/modules/core/repositories/branches'
import { getStream, getStreamCollaborators } from '@/modules/core/repositories/streams'
import { Roles } from '@speckle/shared'
@@ -23,7 +10,7 @@ import { createObject } from '@/modules/core/services/objects'
import { getObject } from '@/modules/core/repositories/objects'
import ObjectLoader from '@speckle/objectloader'
import { noop } from 'lodash'
import { cliLogger } from '@/logging/logging'
import { Logger, crossServerSyncLogger } from '@/logging/logging'
import { createCommitByBranchId } from '@/modules/core/services/commit/management'
import { getUser } from '@/modules/core/repositories/users'
import type { SpeckleViewer } from '@speckle/shared'
@@ -31,6 +18,18 @@ import {
createCommentThreadAndNotify,
createCommentReplyAndNotify
} from '@/modules/comments/services/management'
import {
createApolloClient,
assertValidGraphQLResult
} from '@/modules/cross-server-sync/utils/graphqlClient'
import { CrossServerCommitSyncError } from '@/modules/cross-server-sync/errors'
import {
CrossSyncBranchMetadataQuery,
CrossSyncCommitBranchMetadataQuery,
CrossSyncCommitDownloadMetadataQuery,
CrossSyncDownloadableCommitViewerThreadsQuery,
CrossSyncProjectViewerResourcesQuery
} from '@/modules/cross-server-sync/graph/generated/graphql'
type LocalResources = Awaited<ReturnType<typeof getLocalResources>>
type LocalResourcesWithCommit = LocalResources & { newCommitId: string }
@@ -42,17 +41,14 @@ type ObjectLoaderObject = Record<string, unknown> & {
totalChildrenCount: number
}
type CommitMetadata = Awaited<ReturnType<typeof getCommitMetadata>>
type ViewerThread = Awaited<ReturnType<typeof getViewerThreads>>[0]
const COMMIT_URL_RGX = /((https?:\/\/)?[\w.]+)\/streams\/([\w]+)\/commits\/([\w]+)/i
const MODEL_URL_RGX = /((https?:\/\/)?[\w.]+)\/projects\/([\w]+)\/models\/([\w@,]+)/i
const testQuery = gql`
query CommitDownloadTest {
_
}
`
const commitBranchMetadataQuery = gql`
query CommitBranchMetadata($streamId: String!, $commitId: String!) {
query CrossSyncCommitBranchMetadata($streamId: String!, $commitId: String!) {
stream(id: $streamId) {
commit(id: $commitId) {
id
@@ -63,7 +59,7 @@ const commitBranchMetadataQuery = gql`
`
const branchMetadataQuery = gql`
query BranchMetadata($streamId: String!, $branchName: String!) {
query CrossSyncBranchMetadata($streamId: String!, $branchName: String!) {
stream(id: $streamId) {
branch(name: $branchName) {
id
@@ -73,7 +69,7 @@ const branchMetadataQuery = gql`
`
const commitMetadataQuery = gql`
query CommitDownloadMetadata($streamId: String!, $commitId: String!) {
query CrossSyncCommitDownloadMetadata($streamId: String!, $commitId: String!) {
stream(id: $streamId) {
commit(id: $commitId) {
id
@@ -90,7 +86,10 @@ const commitMetadataQuery = gql`
`
const viewerResourcesQuery = gql`
query ProjectViewerResources($projectId: String!, $resourceUrlString: String!) {
query CrossSyncProjectViewerResources(
$projectId: String!
$resourceUrlString: String!
) {
project(id: $projectId) {
id
viewerResources(resourceIdString: $resourceUrlString) {
@@ -106,7 +105,7 @@ const viewerResourcesQuery = gql`
`
const viewerThreadsQuery = gql`
query DownloadableCommitViewerThreads(
query CrossSyncDownloadableCommitViewerThreads(
$projectId: String!
$filter: ProjectCommentsFilter!
$cursor: String
@@ -139,64 +138,6 @@ const viewerThreadsQuery = gql`
}
`
const assertValidGraphQLResult = (
res: ApolloQueryResult<unknown>,
operationName: string
) => {
if (res.errors?.length) {
throw new Error(
`GQL operation '${operationName}' failed because of errors: ` +
JSON.stringify(res.errors)
)
}
}
const createApolloClient = async (
origin: string,
params?: { token?: string }
): Promise<GraphQLClient> => {
const cache = new InMemoryCache()
const baseLink = new HttpLink({ uri: `${origin}/graphql`, fetch })
const authLink = setContext((_, { headers }) => {
return {
headers: {
...headers,
authorization: params?.token ? `Bearer ${params.token}` : ''
}
}
})
const client = new ApolloClient({
link: authLink.concat(baseLink),
cache,
name: 'cli',
version: getServerVersion(),
defaultOptions: {
query: {
fetchPolicy: 'no-cache',
errorPolicy: 'all'
}
}
})
// Test it out
const res = await client.query({
query: testQuery
})
assertValidGraphQLResult(res, 'Target server test query')
if (!res.data?._) {
throw new Error(
"Couldn't construct working Apollo Client, test query failed cause of unexpected response: " +
JSON.stringify(res.data)
)
}
return client
}
const parseCommitUrl = async (url: string, token?: string) => {
const [, origin, , streamId, commitId] = COMMIT_URL_RGX.exec(url) || []
if (!origin || !streamId || !commitId) {
@@ -253,7 +194,7 @@ const parseIncomingUrl = async (url: string, token?: string) => {
return modelUrl
}
throw new Error(`Couldn't parse commit URL: ${url}`)
throw new CrossServerCommitSyncError(`Couldn't parse commit URL: ${url}`)
}
const getLocalResources = async (
@@ -263,12 +204,14 @@ const getLocalResources = async (
) => {
const targetStream = await getStream({ streamId: targetStreamId })
if (!targetStream) {
throw new Error(`Couldn't find local stream with id ${targetStreamId}`)
throw new CrossServerCommitSyncError(
`Couldn't find local stream with id ${targetStreamId}`
)
}
const targetBranch = await getStreamBranchByName(targetStreamId, branchName)
if (!targetBranch) {
throw new Error(
throw new CrossServerCommitSyncError(
`Couldn't find local branch ${branchName} in stream ${targetStreamId}`
)
}
@@ -285,7 +228,7 @@ const getViewerResources = async (
client: GraphQLClient,
params: { projectId: string; resourceUrlString: string }
) => {
const results = await client.query({
const results = await client.query<CrossSyncProjectViewerResourcesQuery>({
query: viewerResourcesQuery,
variables: params
})
@@ -293,10 +236,12 @@ const getViewerResources = async (
const viewerResources = results.data?.project?.viewerResources
if (!viewerResources) {
throw new Error('Unexpectedly received invalid viewer resources structure')
throw new CrossServerCommitSyncError(
'Unexpectedly received invalid viewer resources structure'
)
}
return viewerResources as ViewerResourceGroup[]
return viewerResources
}
const getCommitBranchId = async (
@@ -304,18 +249,19 @@ const getCommitBranchId = async (
params: { streamId: string; commitId: string }
) => {
const { streamId, commitId } = params
const commitBranchMetadataRes = await client.query({
query: commitBranchMetadataQuery,
variables: { streamId, commitId }
})
const commitBranchMetadataRes =
await client.query<CrossSyncCommitBranchMetadataQuery>({
query: commitBranchMetadataQuery,
variables: { streamId, commitId }
})
assertValidGraphQLResult(commitBranchMetadataRes, 'Commit Branch Metadata Query')
const branchName = commitBranchMetadataRes.data?.stream?.commit?.branchName
if (!branchName) {
throw new Error('Could not resolve commit branch name')
throw new CrossServerCommitSyncError('Could not resolve commit branch name')
}
const branchMetadataRes = await client.query({
const branchMetadataRes = await client.query<CrossSyncBranchMetadataQuery>({
query: branchMetadataQuery,
variables: { streamId, branchName }
})
@@ -323,15 +269,15 @@ const getCommitBranchId = async (
const branchId = branchMetadataRes.data?.stream?.branch?.id
if (!branchId) {
throw new Error('Could not resolve commit branch id')
throw new CrossServerCommitSyncError('Could not resolve commit branch id')
}
return branchId as string
return branchId
}
const getCommitMetadata = async (client: GraphQLClient, params: ParsedCommitUrl) => {
const { streamId, commitId } = params
const results = await client.query({
const results = await client.query<CrossSyncCommitDownloadMetadataQuery>({
query: commitMetadataQuery,
variables: { streamId, commitId }
})
@@ -339,16 +285,18 @@ const getCommitMetadata = async (client: GraphQLClient, params: ParsedCommitUrl)
const commit = results.data?.stream?.commit
if (!commit) {
throw new Error('Unexpectedly received invalid commit structure')
throw new CrossServerCommitSyncError(
'Unexpectedly received invalid commit structure'
)
}
return commit as Commit
return commit
}
const getViewerThreads = async (client: GraphQLClient, params: ParsedCommitUrl) => {
const { streamId, branchId, commitId } = params
const results = await client.query({
const results = await client.query<CrossSyncDownloadableCommitViewerThreadsQuery>({
query: viewerThreadsQuery,
variables: {
projectId: streamId,
@@ -364,10 +312,12 @@ const getViewerThreads = async (client: GraphQLClient, params: ParsedCommitUrl)
const threads = results.data?.project?.commentThreads?.items
if (!threads) {
throw new Error('Unexpectedly received invalid viewer threads structure')
throw new CrossServerCommitSyncError(
'Unexpectedly received invalid viewer threads structure'
)
}
return threads as Comment[]
return threads
}
const cleanViewerState = (
@@ -393,33 +343,40 @@ const cleanViewerState = (
})
const saveNewThreads = async (
threads: Comment[],
localResources: LocalResourcesWithCommit
threads: ViewerThread[],
localResources: LocalResourcesWithCommit,
options?: Partial<{
logger: typeof crossServerSyncLogger
}>
) => {
const { logger = crossServerSyncLogger } = options || {}
const { commentAuthor, targetStream } = localResources
if (!commentAuthor) return
const threadInputs: { originalComment: Comment; input: CreateCommentInput }[] =
threads.map((t) => ({
originalComment: t,
input: {
projectId: targetStream.id,
content: {
doc: t.text.doc,
blobIds: []
},
viewerState: t.viewerState
? cleanViewerState(
t.viewerState as SpeckleViewer.ViewerState.SerializedViewerState,
localResources
)
: null,
screenshot: t.screenshot,
resourceIdString: `${localResources.targetBranch.id}@${localResources.newCommitId}`
}
}))
const threadInputs: { originalComment: ViewerThread; input: CreateCommentInput }[] =
threads
.filter((t) => !!t.text.doc)
.map((t) => ({
originalComment: t,
input: {
projectId: targetStream.id,
content: {
doc: t.text.doc,
blobIds: [] // TODO: Currently not supported
},
viewerState: t.viewerState
? cleanViewerState(
t.viewerState as SpeckleViewer.ViewerState.SerializedViewerState,
localResources
)
: null,
screenshot: t.screenshot,
resourceIdString: `${localResources.targetBranch.id}@${localResources.newCommitId}`
}
}))
if (!threadInputs.length) return
cliLogger.info(`Creating ${threadInputs.length} new comment threads...`)
logger.info(`Creating ${threadInputs.length} new comment threads...`)
const res = await Promise.all(
threadInputs.map((i) =>
createCommentThreadAndNotify(i.input, commentAuthor.id).then((c) => ({
@@ -428,7 +385,7 @@ const saveNewThreads = async (
}))
)
)
cliLogger.info(`...created ${res.length} new comment threads!`)
logger.info(`...created ${res.length} new comment threads!`)
for (const resItem of res) {
const { originalData, newComment } = resItem
@@ -436,7 +393,7 @@ const saveNewThreads = async (
const { replies } = originalComment
if (!replies) continue
cliLogger.info(
logger.info(
`Creating ${replies.items.length} new replies for comment thread ${originalComment.id}...`
)
await Promise.all(
@@ -455,11 +412,14 @@ const saveNewThreads = async (
)
)
)
cliLogger.info(`...created ${replies.items.length} new replies!`)
logger.info(`...created ${replies.items.length} new replies!`)
}
}
const saveNewCommit = async (commit: Commit, localResources: LocalResources) => {
const saveNewCommit = async (
commit: CommitMetadata,
localResources: LocalResources
) => {
const { targetStream, targetBranch, owner } = localResources
const streamId = targetStream.id
@@ -504,10 +464,14 @@ const saveNewCommit = async (commit: Commit, localResources: LocalResources) =>
const createNewObject = async (
newObject: ObjectLoaderObject,
targetStreamId: string
targetStreamId: string,
options?: Partial<{
logger: typeof crossServerSyncLogger
}>
) => {
const { logger = crossServerSyncLogger } = options || {}
if (!newObject) {
cliLogger.error('Encountered falsy object!')
logger.error('Encountered falsy object!')
return
}
@@ -519,17 +483,25 @@ const createNewObject = async (
const newRecord = await getObject(newObjectId, targetStreamId)
if (!newRecord) {
throw new Error("Unexpected error! Just inserted an object, but can't find it!")
throw new CrossServerCommitSyncError(
"Unexpected error! Just inserted an object, but can't find it!"
)
}
return newRecord
}
const loadAllObjectsFromParent = async (params: {
targetStreamId: string
sourceCommit: Commit
parsedCommitUrl: ParsedCommitUrl
}) => {
const loadAllObjectsFromParent = async (
params: {
targetStreamId: string
sourceCommit: CommitMetadata
parsedCommitUrl: ParsedCommitUrl
},
options?: Partial<{
logger: typeof crossServerSyncLogger
}>
) => {
const { logger = crossServerSyncLogger } = options || {}
const {
targetStreamId,
sourceCommit,
@@ -549,29 +521,51 @@ const loadAllObjectsFromParent = async (params: {
let processedObjectCount = 1
for await (const obj of objectLoader.getObjectIterator()) {
const typedObj = obj as ObjectLoaderObject
cliLogger.info(
`Processing ${obj.id} - ${processedObjectCount++}/${totalObjectCount}`
)
await createNewObject(typedObj, targetStreamId)
logger.debug(`Processing ${obj.id} - ${processedObjectCount++}/${totalObjectCount}`)
await createNewObject(typedObj, targetStreamId, { logger })
}
}
/**
* Downloads a commit/version (both FE1 and FE2 supported) from an external Speckle server instance
*/
export const downloadCommit = async (
argv: {
/**
* A FE1 commit URL or an FE2 model/version URL
*/
commitUrl: string
/**
* ID of the local stream that should receive the commit
*/
targetStreamId: string
branchName: string
/**
* Stream branch that should receive the commit. Defaults to 'main'
*/
branchName?: string
/**
* Specify if target commit is private
*/
token?: string
/**
* Specify if you want comments to be pulled in also
*/
commentAuthorId?: string
},
options?: Partial<{
logger: typeof cliLogger
logger: Logger
}>
) => {
const { commitUrl, targetStreamId, branchName, token, commentAuthorId } = argv
const { logger = cliLogger } = options || {}
const {
commitUrl,
targetStreamId,
branchName = 'main',
token,
commentAuthorId
} = argv
const { logger = crossServerSyncLogger } = options || {}
logger.debug(`Process started at: ${new Date().toISOString()}`)
logger.debug(`Commit/version download started at: ${new Date().toISOString()}`)
const localResources = await getLocalResources(
targetStreamId,
@@ -598,16 +592,19 @@ export const downloadCommit = async (
logger.debug(`Created new local commit: ${newCommitId}`)
logger.debug(`Pulling & saving all objects! (${commit.totalChildrenCount})`)
await loadAllObjectsFromParent({
targetStreamId,
sourceCommit: commit,
parsedCommitUrl
})
await loadAllObjectsFromParent(
{
targetStreamId,
sourceCommit: commit,
parsedCommitUrl
},
{ logger }
)
if (localResources.commentAuthor) {
logger.debug(`Pulling & saving all comments w/ #${commentAuthorId} as author!`)
const threads = await getViewerThreads(client, parsedCommitUrl)
await saveNewThreads(threads, newResources)
await saveNewThreads(threads, newResources, { logger })
}
const linkToNewCommit = parsedCommitUrl.isFe2
packages/server/modules/cross-server-sync/services/onboardingProject.ts
+70
View File
@@ -0,0 +1,70 @@
import { crossServerSyncLogger } from '@/logging/logging'
import {
getOnboardingBaseStream,
markOnboardingBaseStream
} from '@/modules/core/repositories/streams'
import { getFirstAdmin } from '@/modules/core/repositories/users'
import { downloadProject } from '@/modules/cross-server-sync/services/project'
import {
getOnboardingStreamCacheBustNumber,
getOnboardingStreamUrl
} from '@/modules/shared/helpers/envHelper'
const getMetadata = () => {
const url = getOnboardingStreamUrl()
const cacheBustNumber = getOnboardingStreamCacheBustNumber()
if (!url) return null
const version = `${url}:::${cacheBustNumber}`
return { url, cacheBustNumber, version }
}
export async function getOnboardingBaseProject() {
const metadata = getMetadata()
if (!metadata) {
return undefined
}
return await getOnboardingBaseStream(metadata.version)
}
export async function ensureOnboardingProject() {
const logger = crossServerSyncLogger
logger.info('Ensuring onboarding project is present...')
const metadata = getMetadata()
if (!metadata) {
logger.info('No base onboarding stream configured through env vars...')
return undefined
}
const [existingStream, admin] = await Promise.all([
getOnboardingBaseStream(metadata.version),
getFirstAdmin()
])
if (existingStream) {
logger.debug('Onboarding stream already exists, skipping...')
return existingStream
}
if (!admin) {
logger.info('No admin user found, skipping onboarding stream creation...')
return undefined
}
logger.debug('Onboarding stream not found, pulling from target server...')
const res = await downloadProject(
{
projectUrl: metadata.url,
authorId: admin.id,
syncComments: true
},
{ logger }
)
logger.debug('Marking stream as onboarding base...')
await markOnboardingBaseStream(res.projectId, metadata.version)
logger.info('Onboarding base stream created successfully!')
return res.project
}
packages/server/modules/cross-server-sync/services/project.ts
+227
View File
@@ -0,0 +1,227 @@
import { crossServerSyncLogger, Logger } from '@/logging/logging'
import { getUser } from '@/modules/core/repositories/users'
import { CrossServerProjectSyncError } from '@/modules/cross-server-sync/errors'
import {
createApolloClient,
GraphQLClient,
gql,
assertValidGraphQLResult
} from '@/modules/cross-server-sync/utils/graphqlClient'
import { CrossSyncProjectMetadataQuery } from '@/modules/cross-server-sync/graph/generated/graphql'
import { omit } from 'lodash'
import { downloadCommit } from '@/modules/cross-server-sync/services/commit'
import { getFrontendOrigin } from '@/modules/shared/helpers/envHelper'
import { createStreamReturnRecord } from '@/modules/core/services/streams/management'
import { createBranchAndNotify } from '@/modules/core/services/branch/management'
import { getStreamBranchByName } from '@/modules/core/repositories/branches'
type ProjectMetadata = Awaited<ReturnType<typeof getProjectMetadata>>
const PROJECT_URL_RGX = /((https?:\/\/)?[\w.]+)\/projects\/([\w]+)\/?/i
const projectMetadataQuery = gql`
query CrossSyncProjectMetadata($id: String!, $versionsCursor: String) {
project(id: $id) {
id
name
description
visibility
versions(limit: 100, cursor: $versionsCursor) {
totalCount
cursor
items {
id
createdAt
model {
id
name
}
}
}
}
}
`
const getLocalResources = async (params: { authorId: string }) => {
const { authorId } = params
const user = await getUser(authorId)
if (!user) {
throw new CrossServerProjectSyncError('Target author not found')
}
return { user }
}
const parseIncomingUrl = (projectUrl: string) => {
const [, origin, , projectId] = PROJECT_URL_RGX.exec(projectUrl) || []
if (!origin || !projectId) {
throw new CrossServerProjectSyncError('Invalid project URL')
}
return { origin, projectId }
}
const getProjectMetadata = async (params: {
client: GraphQLClient
projectId: string
}) => {
const { client, projectId } = params
// Load 1st page
const res = await client.query<CrossSyncProjectMetadataQuery>({
query: projectMetadataQuery,
variables: {
id: projectId
}
})
assertValidGraphQLResult(res, 'Project metadata query')
const projectInfo = omit(res.data.project, ['versions'])
const versions = res.data.project.versions.items
// Load all pages of versions
let cursor = res.data.project.versions.cursor
let failsafe = 10
while (cursor && failsafe-- > 0) {
const res = await client.query<CrossSyncProjectMetadataQuery>({
query: projectMetadataQuery,
variables: {
id: projectId,
versionsCursor: cursor
}
})
assertValidGraphQLResult(res, 'Project metadata query')
versions.push(...res.data.project.versions.items)
cursor = res.data.project.versions.cursor
}
// Sort versions by descending creation data
versions.sort(
(a, b) => new Date(a.createdAt).getTime() - new Date(b.createdAt).getTime()
)
return { projectInfo, versions }
}
const ensureBranch = async (params: {
streamId: string
branchName: string
authorId: string
}) => {
const { streamId, branchName, authorId } = params
const existingBranch = await getStreamBranchByName(streamId, branchName)
if (!existingBranch) {
const newBranch = await createBranchAndNotify(
{
streamId,
name: branchName
},
authorId
)
return newBranch
}
return existingBranch
}
const importVersions = async (params: {
logger: Logger
projectInfo: ProjectMetadata
localProjectId: string
localAuthorId: string
origin: string
syncComments?: boolean
}) => {
const { logger, projectInfo, origin, localProjectId, syncComments, localAuthorId } =
params
const projectId = projectInfo.projectInfo.id
logger.debug(`Serially downloading ${projectInfo.versions.length} versions...`)
for (const version of projectInfo.versions) {
// Ensure branch exists
const branchName = version.model.name
await ensureBranch({
streamId: localProjectId,
branchName,
authorId: localAuthorId
})
// Actually download
const url = new URL(
`/projects/${projectId}/models/${version.model.id}@${version.id}`,
origin
)
await downloadCommit(
{
commitUrl: url.toString(),
targetStreamId: localProjectId,
commentAuthorId: syncComments ? localAuthorId : undefined,
branchName
},
{ logger }
)
}
}
/**
* Downloads a project from an external FE2 Speckle server instance
*/
export const downloadProject = async (
params: {
/**
* An FE2 project URL (must be publicly accessible)
*/
projectUrl: string
/**
* ID of user that should own the project locally
*/
authorId: string
syncComments?: boolean
},
options?: Partial<{
logger: Logger
}>
) => {
const { projectUrl, authorId, syncComments } = params
const { logger = crossServerSyncLogger } = options || {}
logger.info(`Project download started at: ${new Date().toISOString()}`)
const localResources = await getLocalResources({ authorId })
const parsedUrl = parseIncomingUrl(projectUrl)
const client = await createApolloClient(parsedUrl.origin)
logger.debug(`Resolving project metadata and associated versions...`)
const projectInfo = await getProjectMetadata({
client,
projectId: parsedUrl.projectId
})
logger.debug(`Creating project locally...`)
const project = await createStreamReturnRecord({
...projectInfo.projectInfo,
ownerId: localResources.user.id
})
await importVersions({
logger,
projectInfo,
localProjectId: project.id,
localAuthorId: localResources.user.id,
origin: parsedUrl.origin,
syncComments
})
logger.info(`Project download completed at: ${new Date().toISOString()}`)
const newProjectUrl = new URL(
`/projects/${project.id}`,
getFrontendOrigin(true)
).toString()
logger.info(`New Project URL: ${newProjectUrl}`)
return {
newProjectUrl,
projectId: project.id,
project
}
}
packages/server/modules/cross-server-sync/utils/graphqlClient.ts
+79
View File
@@ -0,0 +1,79 @@
import {
ApolloClient,
InMemoryCache,
NormalizedCacheObject,
HttpLink,
gql,
ApolloQueryResult
} from '@apollo/client/core'
import { setContext } from '@apollo/client/link/context'
import { getServerVersion } from '@/modules/shared/helpers/envHelper'
import { CrossSyncClientTestQuery } from '@/modules/cross-server-sync/graph/generated/graphql'
export type GraphQLClient = ApolloClient<NormalizedCacheObject>
const testQuery = gql`
query CrossSyncClientTest {
_
}
`
export const assertValidGraphQLResult = (
res: ApolloQueryResult<unknown>,
operationName: string
) => {
if (res.errors?.length) {
throw new Error(
`GQL operation '${operationName}' failed because of errors: ` +
JSON.stringify(res.errors)
)
}
}
export const createApolloClient = async (
origin: string,
params?: { token?: string }
): Promise<GraphQLClient> => {
const cache = new InMemoryCache()
const baseLink = new HttpLink({ uri: `${origin}/graphql`, fetch })
const authLink = setContext((_, { headers }) => {
return {
headers: {
...headers,
authorization: params?.token ? `Bearer ${params.token}` : ''
}
}
})
const client = new ApolloClient({
link: authLink.concat(baseLink),
cache,
name: 'cli',
version: getServerVersion(),
defaultOptions: {
query: {
fetchPolicy: 'no-cache',
errorPolicy: 'all'
}
}
})
// Test it out
const res = await client.query<CrossSyncClientTestQuery>({
query: testQuery
})
assertValidGraphQLResult(res, 'Target server test query')
if (!res.data?._) {
throw new Error(
"Couldn't construct working Apollo Client, test query failed cause of unexpected response: " +
JSON.stringify(res.data)
)
}
return client
}
export { gql }
packages/server/modules/index.js
+3 -2
View File
@@ -56,7 +56,8 @@ async function getSpeckleModules() {
'./notifications',
'./activitystream',
'./accessrequests',
'./webhooks'
'./webhooks',
'./cross-server-sync'
]
for (const dir of moduleDirs) {
@@ -72,7 +73,7 @@ exports.init = async (app) => {
// Stage 1: initialise all modules
for (const module of modules) {
await module.init(app, isInitial)
await module.init?.(app, isInitial)
}
// Stage 2: finalize init all modules
packages/server/modules/shared/helpers/envHelper.ts
+25 -8
View File
@@ -1,4 +1,3 @@
import { Nullable } from '@speckle/shared'
import { MisconfiguredEnvironmentError } from '@/modules/shared/errors'
import { trimEnd } from 'lodash'
@@ -159,13 +158,6 @@ export function isSSLServer() {
return /^https:\/\//.test(getBaseUrl())
}
/**
* Source stream for cloning tutorial/guide streams for users
*/
export function getOnboardingStreamId(): Nullable<string> {
return process.env.ONBOARDING_STREAM_ID || null
}
export function adminOverrideEnabled() {
return process.env.ADMIN_OVERRIDE_ENABLED === 'true'
}
@@ -187,3 +179,28 @@ export function speckleAutomateUrl() {
export function ignoreMissingMigrations() {
return ['1', 'true'].includes(process.env.IGNORE_MISSING_MIRATIONS || 'false')
}
/**
* URL of a project on any FE2 speckle server that will be pulled in and used as the onboarding stream
*/
export function getOnboardingStreamUrl() {
const val = process.env.ONBOARDING_STREAM_URL
if (!val?.length) return null
try {
// validating that the URL is valid
return new URL(val).toString()
} catch (e) {
// suppress
}
return null
}
/**
* Increase this value to re-sync the onboarding stream
*/
export function getOnboardingStreamCacheBustNumber() {
const val = process.env.ONBOARDING_STREAM_CACHE_BUST_NUMBER || '1'
return parseInt(val) || 1
}
packages/server/modules/shared/helpers/typeHelper.ts
+1 -1
View File
@@ -24,7 +24,7 @@ export type SpeckleModule<T extends Record<string, unknown> = Record<string, unk
* @param isInitial Whether this initialization method is being invoked for the first time in this
* process. In tests modules can be initialized multiple times.
*/
init: (app: Express, isInitial: boolean) => MaybeAsync<void>
init?: (app: Express, isInitial: boolean) => MaybeAsync<void>
/**
* Finalize initialization. This is only invoked once all of the other modules' `init()`
* hooks are run.
packages/server/package.json
+4 -4
View File
@@ -12,7 +12,7 @@
"url": "https://github.com/specklesystems/Server.git"
},
"engines": {
"node": "^18.16.1"
"node": "^18.17.0"
},
"scripts": {
"build": "tsc -p ./tsconfig.build.json",
@@ -32,6 +32,7 @@
"gqlgen:watch": "graphql-codegen --config codegen.yml --watch \"assets/**/*.graphql\""
},
"dependencies": {
"@apollo/client": "^3.7.0",
"@aws-sdk/client-s3": "^3.276.0",
"@aws-sdk/lib-storage": "^3.100.0",
"@godaddy/terminus": "^4.9.0",
@@ -39,6 +40,7 @@
"@mailchimp/mailchimp_marketing": "^3.0.80",
"@sentry/node": "^6.17.9",
"@sentry/tracing": "^6.17.9",
"@speckle/objectloader": "workspace:^",
"@speckle/shared": "workspace:^",
"@types/mailchimp__mailchimp_marketing": "^3.0.9",
"@types/pino-http": "^5.8.1",
@@ -50,6 +52,7 @@
"compression": "^1.7.4",
"connect-redis": "^6.1.1",
"cors": "^2.8.5",
"cross-fetch": "^3.1.5",
"crypto-random-string": "^3.2.0",
"dataloader": "^2.0.0",
"dayjs": "^1.11.5",
@@ -99,7 +102,6 @@
"zxcvbn": "^4.4.2"
},
"devDependencies": {
"@apollo/client": "^3.7.0",
"@apollo/rover": "^0.14.1",
"@bull-board/express": "^4.2.2",
"@faker-js/faker": "^7.1.0",
@@ -107,7 +109,6 @@
"@graphql-codegen/typescript": "2.7.2",
"@graphql-codegen/typescript-operations": "^2.5.2",
"@graphql-codegen/typescript-resolvers": "2.7.2",
"@speckle/objectloader": "workspace:^",
"@swc/core": "^1.2.222",
"@tiptap/core": "^2.0.0-beta.176",
"@types/bcrypt": "^5.0.0",
@@ -138,7 +139,6 @@
"chai-http": "^4.3.0",
"concurrently": "^7.0.0",
"cross-env": "^7.0.3",
"cross-fetch": "^3.1.5",
"deep-equal-in-any-order": "^1.1.15",
"eslint": "^8.11.0",
"eslint-config-prettier": "^8.5.0",
packages/shared/package.json
+1 -1
View File
@@ -22,7 +22,7 @@
},
"sideEffects": false,
"engines": {
"node": "^18.16.1"
"node": "^18.17.0"
},
"author": "AEC Systems",
"license": "Apache-2.0",
packages/viewer-sandbox/src/Sandbox.ts
+10 -4
View File
@@ -982,16 +982,16 @@ export default class Sandbox {
diffResult = await this.viewer.diff(
//building
// 'https://latest.speckle.dev/streams/aea12cab71/objects/bcf37136dea9fe9397cdfd84012f616a',
// 'https://latest.speckle.dev/streams/aea12cab71/objects/94af0a6b4eaa318647180f8c230cb867'
// 'https://latest.speckle.dev/streams/aea12cab71/objects/94af0a6b4eaa318647180f8c230cb867',
// cubes
// 'https://latest.speckle.dev/streams/aea12cab71/objects/d2510c59c203b73473f8bbfe637e0552',
// 'https://latest.speckle.dev/streams/aea12cab71/objects/1c327da824fdb04629eb48675101d7b7',
// sketchup
// 'https://latest.speckle.dev/streams/aea12cab71/objects/06bed1819e6c61d9df7196d424ab1eec',
// 'https://latest.speckle.dev/streams/aea12cab71/objects/9026f1d6495789b9eab31b5028c9a8ef'
// 'https://latest.speckle.dev/streams/aea12cab71/objects/9026f1d6495789b9eab31b5028c9a8ef',
//latest
'https://latest.speckle.dev/streams/cdbe82b016/objects/c14d1a33fd68323193813ec215737472',
'https://latest.speckle.dev/streams/cdbe82b016/objects/16676fc95a9ead877f6a825d9e28cbe8',
// 'https://latest.speckle.dev/streams/cdbe82b016/objects/c14d1a33fd68323193813ec215737472',
// 'https://latest.speckle.dev/streams/cdbe82b016/objects/16676fc95a9ead877f6a825d9e28cbe8',
//lines
// 'https://latest.speckle.dev/streams/92b620fb17/objects/3b42d6ef51d3110b4e33b9f8cdc9f357',
// 'https://latest.speckle.dev/streams/92b620fb17/objects/774384d431fb34d447d4696abbc4b816',
@@ -1016,6 +1016,12 @@ export default class Sandbox {
// bug
// 'https://latest.speckle.dev/streams/0c6ad366c4/objects/03f0a8bf0ed8064865eda87a865c7212',
// 'https://latest.speckle.dev/streams/0c6ad366c4/objects/33ef6b9b547dc9688eb40157b967eab9',
// large
'https://speckle.xyz/streams/e6f9156405/objects/650f358d8aac50168d9e9226ef6f5cbc',
'https://latest.speckle.dev/streams/92b620fb17/objects/1154ca1d997ac631571db55f84cb703d',
// cubes
// 'https://latest.speckle.dev/streams/0c6ad366c4/objects/03f0a8bf0ed8064865eda87a865c7212',
// 'https://latest.speckle.dev/streams/0c6ad366c4/objects/33ef6b9b547dc9688eb40157b967eab9',
VisualDiffMode.COLORED,
localStorage.getItem('AuthTokenLatest') as string
packages/viewer-sandbox/src/main.ts
+3 -2
View File
@@ -110,7 +110,7 @@ const getStream = () => {
// prettier-ignore
// 'https://speckle.xyz/streams/da9e320dad/commits/5388ef24b8?c=%5B-7.66134,10.82932,6.41935,-0.07739,-13.88552,1.8697,0,1%5D'
// Revit sample house (good for bim-like stuff with many display meshes)
'https://speckle.xyz/streams/da9e320dad/commits/5388ef24b8'
// 'https://speckle.xyz/streams/da9e320dad/commits/5388ef24b8'
// 'https://latest.speckle.dev/streams/c1faab5c62/commits/6c6e43e5f3'
// 'https://latest.speckle.dev/streams/58b5648c4d/commits/60371ecb2d'
// 'Super' heavy revit shit
@@ -278,11 +278,12 @@ const getStream = () => {
// 'https://latest.speckle.dev/streams/b68abcbf2e/commits/4e94ecad62'
// Big ass mafa'
// 'https://speckle.xyz/streams/88307505eb/objects/a232d760059046b81ff97e6c4530c985'
// 'https://latest.speckle.dev/streams/92b620fb17/commits/dfb9ca025d'
'https://latest.speckle.dev/streams/92b620fb17/commits/dfb9ca025d'
// 'Blocks with elements
// 'https://latest.speckle.dev/streams/e258b0e8db/commits/00e165cc1c'
// 'https://latest.speckle.dev/streams/e258b0e8db/commits/e48cf53add'
// 'https://latest.speckle.dev/streams/e258b0e8db/commits/c19577c7d6?c=%5B15.88776,-8.2182,12.17095,18.64059,1.48552,0.6025,0,1%5D'
// 'https://speckle.xyz/streams/46caea9b53/commits/71938adcd1'
)
}
packages/viewer/package.json
+3 -2
View File
@@ -30,7 +30,7 @@
"dist"
],
"engines": {
"node": "^18.16.1"
"node": "^18.17.0"
},
"scripts": {
"build": "NODE_ENV=production rollup --config",
@@ -61,7 +61,8 @@
"three": "^0.140.0",
"three-mesh-bvh": "0.5.17",
"tree-model": "1.0.7",
"troika-three-text": "0.47.2"
"troika-three-text": "0.47.2",
"underscore": "1.13.6"
},
"devDependencies": {
"@babel/core": "^7.18.2",
packages/viewer/src/modules/Differ.ts
+139 -31
View File
@@ -1,3 +1,4 @@
/* eslint-disable @typescript-eslint/no-unused-vars */
import { Color, FrontSide } from 'three'
import { SpeckleTypeAllRenderables } from './converter/GeometryConverter'
import SpeckleStandardMaterial from './materials/SpeckleStandardMaterial'
@@ -7,6 +8,7 @@ import { GeometryType } from './batching/Batch'
import SpeckleLineMaterial from './materials/SpeckleLineMaterial'
import Logger from 'js-logger'
import { NodeRenderView } from './tree/NodeRenderView'
import _, { omit } from 'underscore'
// eslint-disable-next-line @typescript-eslint/no-explicit-any
type SpeckleObject = Record<string, any>
@@ -184,10 +186,41 @@ export class Differ {
this.removedMaterialPoint.toneMapped = false
}
public diff(urlA: string, urlB: string): Promise<DiffResult> {
const modifiedNew: Array<SpeckleObject> = []
const modifiedOld: Array<SpeckleObject> = []
private intersection(o1, o2) {
const [k1, k2] = [Object.keys(o1), Object.keys(o2)]
const [first, next] = k1.length > k2.length ? [k2, o1] : [k1, o2]
return first.filter((k) => k in next)
}
private buildIdMaps(
rvs: Array<TreeNode>,
idMap: { [id: string]: { node: TreeNode; applicationId: string } },
appIdMap: { [id: string]: number }
) {
for (let k = 0; k < rvs.length; k++) {
const atomicRv = rvs[k]
const applicationId = atomicRv.model.raw.applicationId
? atomicRv.model.raw.applicationId
: this.tree
.getAncestors(atomicRv)
.find((value) => value.model.raw.applicationId)?.model.raw.applicationId
idMap[atomicRv.model.raw.id] = {
node: atomicRv,
applicationId
}
if (applicationId) {
appIdMap[applicationId] = 1
}
}
}
public diff(urlA: string, urlB: string): Promise<DiffResult> {
return this.diffIterative(urlA, urlB)
}
private diffBoolean(urlA: string, urlB: string): Promise<DiffResult> {
const start = performance.now()
const diffResult: DiffResult = {
unchanged: [],
added: [],
@@ -197,8 +230,6 @@ export class Differ {
const renderTreeA = this.tree.getRenderTree(urlA)
const renderTreeB = this.tree.getRenderTree(urlB)
const rootA = this.tree.findId(urlA)
const rootB = this.tree.findId(urlB)
let rvsA = renderTreeA.getRenderableNodes(...SpeckleTypeAllRenderables)
let rvsB = renderTreeB.getRenderableNodes(...SpeckleTypeAllRenderables)
@@ -213,18 +244,106 @@ export class Differ {
rvsA = [...Array.from(new Set(rvsA))]
rvsB = [...Array.from(new Set(rvsB))]
const idMapA = {}
const appIdMapA = {}
this.buildIdMaps(rvsA, idMapA, appIdMapA)
const idMapB = {}
const appIdMapB = {}
this.buildIdMaps(rvsB, idMapB, appIdMapB)
/** Get the ids which are common between the two maps. This will be objects
* which have not changed
*/
const unchanged: Array<string> = this.intersection(idMapA, idMapB)
/** We remove the unchanged objects from B and end up with changed + added */
const addedModified = _.omit(idMapB, unchanged)
/** We remove the unchanged objects from A and end up with changed + removed */
const removedModified = _.omit(idMapA, unchanged)
/** We remove the changed objects from B. An object from B is changed if
* it's application ID exists in A
*/
const added = _.omit(addedModified, function (value, key, object) {
return value.applicationId && appIdMapA[value.applicationId] !== undefined
})
/** We remove the changed objects from A. An object from A is changed if
* it's application ID exists in B
*/
const removed = _.omit(removedModified, function (value, key, object) {
return value.applicationId && appIdMapB[value.applicationId] !== undefined
})
/** We remove the removed objects from A, leaving us only changed objects */
const modifiedRemoved = _.omit(removedModified, Object.keys(removed))
/** We remove the removed objects from B, leaving us only changed objects */
const modifiedAdded = _.omit(addedModified, Object.keys(added))
/** We fill the arrays from here on out */
const modifiedOld = Object.values(modifiedRemoved).map(
(value: { node: TreeNode }) => value.node
)
const modifiedNew = Object.values(modifiedAdded).map(
(value: { node: TreeNode }) => value.node
)
diffResult.unchanged.push(...unchanged.map((value) => idMapA[value].node))
diffResult.unchanged.push(...unchanged.map((value) => idMapB[value].node))
diffResult.removed.push(
...Object.values(removed).map((value: { node: TreeNode }) => value.node)
)
diffResult.added.push(
...Object.values(added).map((value: { node: TreeNode }) => value.node)
)
modifiedOld.forEach((value, index) => {
value
diffResult.modified.push([modifiedOld[index], modifiedNew[index]])
})
console.warn('Boolean Time -> ', performance.now() - start)
return Promise.resolve(diffResult)
}
private diffIterative(urlA: string, urlB: string): Promise<DiffResult> {
const start = performance.now()
const modifiedNew: Array<SpeckleObject> = []
const modifiedOld: Array<SpeckleObject> = []
const diffResult: DiffResult = {
unchanged: [],
added: [],
removed: [],
modified: []
}
const renderTreeA = this.tree.getRenderTree(urlA)
const renderTreeB = this.tree.getRenderTree(urlB)
let rvsA = renderTreeA.getRenderableNodes(...SpeckleTypeAllRenderables)
let rvsB = renderTreeB.getRenderableNodes(...SpeckleTypeAllRenderables)
rvsA = rvsA.map((value) => {
return renderTreeA.getAtomicParent(value)
})
rvsB = rvsB.map((value) => {
return renderTreeB.getAtomicParent(value)
})
rvsA = [...Array.from(new Set(rvsA))]
rvsB = [...Array.from(new Set(rvsB))]
const idMapA = {}
const appIdMapA = {}
this.buildIdMaps(rvsA, idMapA, appIdMapA)
const idMapB = {}
const appIdMapB = {}
this.buildIdMaps(rvsB, idMapB, appIdMapB)
for (let k = 0; k < rvsB.length; k++) {
const res = rootA.first((node: TreeNode) => {
return rvsB[k].model.raw.id === node.model.raw.id
})
const res = idMapA[rvsB[k].model.raw.id]?.node
if (res) {
diffResult.unchanged.push(res)
} else {
const applicationId = rvsB[k].model.raw.applicationId
? rvsB[k].model.raw.applicationId
: this.tree
.getAncestors(rvsB[k])
.find((value) => value.model.raw.applicationId)
const applicationId = idMapB[rvsB[k].model.raw.id].applicationId
if (!applicationId) {
Logger.error(
`No application ID found. Object id:${rvsB[k].model.raw.id} is considered 'added'!`
@@ -232,9 +351,7 @@ export class Differ {
diffResult.added.push(rvsB[k])
continue
}
const res2 = rootA.first((node: TreeNode) => {
return applicationId === node.model.raw.applicationId
})
const res2 = appIdMapA[applicationId]
if (res2) {
modifiedNew.push(rvsB[k])
} else {
@@ -242,17 +359,10 @@ export class Differ {
}
}
}
for (let k = 0; k < rvsA.length; k++) {
const res = rootB.first((node: TreeNode) => {
return rvsA[k].model.raw.id === node.model.raw.id
})
const res = idMapB[rvsA[k].model.raw.id]?.node
if (!res) {
const applicationId = rvsA[k].model.raw.applicationId
? rvsA[k].model.raw.applicationId
: this.tree
.getAncestors(rvsA[k])
.find((value) => value.model.raw.applicationId)
const applicationId = idMapA[rvsA[k].model.raw.id].applicationId
if (!applicationId) {
Logger.error(
`No application ID found. Object id:${rvsA[k].model.raw.id} is considered 'removed'!`
@@ -260,9 +370,7 @@ export class Differ {
diffResult.removed.push(rvsA[k])
continue
}
const res2 = rootB.first((node: TreeNode) => {
return applicationId === node.model.raw.applicationId
})
const res2 = appIdMapB[applicationId]
if (!res2) {
diffResult.removed.push(rvsA[k])
} else {
@@ -272,13 +380,11 @@ export class Differ {
diffResult.unchanged.push(res)
}
}
modifiedOld.forEach((value, index) => {
value
diffResult.modified.push([modifiedOld[index], modifiedNew[index]])
})
console.warn(diffResult)
console.warn('Interative Time -> ', performance.now() - start)
return Promise.resolve(diffResult)
}
@@ -322,6 +428,7 @@ export class Differ {
[id: string]: SpeckleStandardMaterial | SpecklePointMaterial | SpeckleLineMaterial
}
) {
const start = performance.now()
switch (mode) {
case VisualDiffMode.COLORED:
this._materialGroups = this.getColoredMaterialGroups(
@@ -337,6 +444,7 @@ export class Differ {
default:
Logger.error(`Unsupported visual diff mode ${mode}`)
}
console.warn('Material groups -> ', performance.now() - start)
return this._materialGroups
}
packages/viewer/src/modules/SpeckleRenderer.ts
+18 -2
View File
@@ -77,6 +77,7 @@ export enum ObjectLayers {
export default class SpeckleRenderer {
private readonly SHOW_HELPERS = false
private readonly IGNORE_ZERO_OPACITY_OBJECTS = true
public SHOW_BVH = false
private container: HTMLElement
private _renderer: WebGLRenderer
@@ -890,13 +891,28 @@ export default class SpeckleRenderer {
const rvs = []
const points = []
for (let k = 0; k < results.length; k++) {
let rv = results[k].batchObject?.renderView
if (!rv) {
const batchObject = results[k].batchObject
let rv = null
if (batchObject) {
rv = batchObject.renderView
const material = (results[k].object as SpeckleMesh).getBatchObjectMaterial(
results[k].batchObject
)
if (material.opacity === 0 && this.IGNORE_ZERO_OPACITY_OBJECTS) continue
} else {
rv = this.batcher.getRenderView(
results[k].object.uuid,
results[k].faceIndex !== undefined ? results[k].faceIndex : results[k].index
)
if (rv) {
const material = this.batcher.getRenderViewMaterial(
results[k].object.uuid,
results[k].faceIndex !== undefined ? results[k].faceIndex : results[k].index
)
if (material.opacity === 0 && this.IGNORE_ZERO_OPACITY_OBJECTS) continue
}
}
if (rv) {
rvs.push(rv)
points.push(results[k].point)
packages/viewer/src/modules/batching/Batch.ts
+1
View File
@@ -31,6 +31,7 @@ export interface Batch {
resetDrawRanges()
buildBatch()
getRenderView(index: number): NodeRenderView
getMaterialAtIndex(index: number): Material
onUpdate(deltaTime: number)
onRender(renderer: WebGLRenderer)
purge()
packages/viewer/src/modules/batching/Batcher.ts
+11 -2
View File
@@ -460,6 +460,15 @@ export default class Batcher {
return this.batches[batchId].getRenderView(index)
}
public getRenderViewMaterial(batchId: string, index: number) {
if (!this.batches[batchId]) {
Logger.error('Invalid batch id!')
return null
}
return this.batches[batchId].getMaterialAtIndex(index)
}
public resetBatchesDrawRanges() {
for (const k in this.batches) {
this.batches[k].resetDrawRanges()
@@ -576,7 +585,7 @@ export default class Batcher {
if (k !== rv.batchId) {
this.batches[k].setDrawRanges({
offset: 0,
count: Infinity,
count: this.batches[k].getCount(),
material: this.materials.getFilterMaterial(
this.batches[k].renderViews[0],
FilterMaterialType.GHOST
@@ -591,7 +600,7 @@ export default class Batcher {
if (k !== batchId) {
this.batches[k].setDrawRanges({
offset: 0,
count: Infinity,
count: this.batches[k].getCount(),
material: this.materials.getFilterMaterial(
this.batches[k].renderViews[0],
FilterMaterialType.GHOST
packages/viewer/src/modules/batching/LineBatch.ts
+6
View File
@@ -5,6 +5,7 @@ import {
InstancedInterleavedBuffer,
InterleavedBufferAttribute,
Line,
Material,
Object3D,
Vector4,
WebGLRenderer
@@ -253,6 +254,11 @@ export default class LineBatch implements Batch {
}
}
public getMaterialAtIndex(index: number): Material {
index
return this.batchMaterial
}
private makeLineGeometry(position: Float64Array) {
this.geometry = this.makeLineGeometryTriangle(new Float32Array(position))
Geometry.updateRTEGeometry(this.geometry, position)
packages/viewer/src/modules/batching/PointBatch.ts
+27
View File
@@ -17,6 +17,7 @@ import {
} from './Batch'
import { GeometryConverter } from '../converter/GeometryConverter'
import { ObjectLayers } from '../SpeckleRenderer'
import Logger from 'js-logger'
export default class PointBatch implements Batch {
public id: string
@@ -353,6 +354,32 @@ export default class PointBatch implements Batch {
}
}
public getMaterialAtIndex(index: number): Material {
for (let k = 0; k < this.renderViews.length; k++) {
if (
index >= this.renderViews[k].batchStart &&
index < this.renderViews[k].batchEnd
) {
const rv = this.renderViews[k]
const group = this.geometry.groups.find((value) => {
return (
rv.batchStart >= value.start &&
rv.batchStart + rv.batchCount <= value.count + value.start
)
})
if (!Array.isArray(this.mesh.material)) {
return this.mesh.material
} else {
if (!group) {
Logger.warn(`Malformed material index!`)
return null
}
return this.mesh.material[group.materialIndex]
}
}
}
}
private makePointGeometry(
position: Float64Array,
color: Float32Array
packages/viewer/src/modules/batching/TextBatch.ts
+1 -3
View File
@@ -131,9 +131,7 @@ export default class TextBatch implements Batch {
}
public getMaterialAtIndex(index: number): Material {
index
console.warn('Deprecated! Do not call this anymore')
return null
return this.batchMaterial
}
public purge() {
packages/webhook-service/Dockerfile
+2 -2
View File
@@ -1,6 +1,6 @@
ARG NODE_ENV=production
FROM node:18.16.1-bullseye-slim as build-stage
FROM node:18.17.0-bullseye-slim as build-stage
ARG NODE_ENV
ENV NODE_ENV=${NODE_ENV}
@@ -32,7 +32,7 @@ ENV TINI_VERSION=${TINI_VERSION}
ADD https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini ./tini
RUN chmod +x ./tini
FROM node:18.16.1-bullseye-slim as dependency-stage
FROM node:18.17.0-bullseye-slim as dependency-stage
# yarn install
ARG NODE_ENV
ENV NODE_ENV=${NODE_ENV}
packages/webhook-service/package.json
+1 -1
View File
@@ -13,7 +13,7 @@
},
"homepage": "https://github.com/specklesystems/speckle-server#readme",
"engines": {
"node": "^18.16.1"
"node": "^18.17.0"
},
"scripts": {
"test": "echo \"Error: no test specified\" && exit 1",
utils/helm/speckle-server/templates/frontend_2/deployment.yml
-1
View File
@@ -67,7 +67,6 @@ spec:
- name: NUXT_PUBLIC_LOG_CLIENT_API_ENDPOINT
value: {{ .Values.frontend_2.logClientApiEndpoint }}
priorityClassName: high-priority
{{- if .Values.frontend_2.affinity }}
affinity: {{- include "speckle.renderTpl" (dict "value" .Values.frontend_2.affinity "context" $) | nindent 8 }}
utils/helm/speckle-server/templates/server/deployment.yml
+4 -2
View File
@@ -108,8 +108,10 @@ spec:
value: http://{{ .Values.domain }}
{{- end }}
- name: ONBOARDING_STREAM_ID
value: {{ .Values.server.onboardingStreamId }}
- name: ONBOARDING_STREAM_URL
value: {{ .Values.server.onboarding.stream_url }}
- name: ONBOARDING_STREAM_CACHE_BUST_NUMBER
value: {{ .Values.server.onboarding.stream_cache_bust_number | quote }}
- name: SESSION_SECRET
valueFrom:
utils/helm/speckle-server/values.schema.json
+15
View File
@@ -466,6 +466,21 @@
"description": "The minimum level of logs which will be output. Suitable values are trace, debug, info, warn, error, fatal, or silent",
"default": "info"
},
"onboarding": {
"type": "object",
"properties": {
"stream_url": {
"type": "string",
"description": "The (cross-server) URL to the project/stream that should be used as the onboarding project base.",
"default": "https://latest.speckle.systems/projects/843d07eb10"
},
"stream_cache_bust_number": {
"type": "number",
"description": "Increase this number to trigger the re-pulling of the base stream",
"default": 1
}
}
},
"inspect": {
"type": "object",
"properties": {
utils/helm/speckle-server/values.yaml
+7 -1
View File
@@ -376,7 +376,13 @@ server:
## @param server.logLevel The minimum level of logs which will be output. Suitable values are trace, debug, info, warn, error, fatal, or silent
##
logLevel: 'info'
onboarding:
## @param server.onboarding.stream_url The (cross-server) URL to the project/stream that should be used as the onboarding project base.
##
stream_url: 'https://latest.speckle.systems/projects/843d07eb10'
## @param server.onboarding.stream_cache_bust_number Increase this number to trigger the re-pulling of the base stream
##
stream_cache_bust_number: 1
inspect:
## @param server.inspect.enabled If enabled, indicates that the Speckle server should be deployed with the nodejs inspect feature enabled
enabled: false
yarn.lock
+8
View File
@@ -12881,6 +12881,7 @@ __metadata:
tree-model: 1.0.7
troika-three-text: 0.47.2
typescript: ^4.5.4
underscore: 1.13.6
languageName: unknown
linkType: soft
@@ -43510,6 +43511,13 @@ __metadata:
languageName: node
linkType: hard
"underscore@npm:1.13.6":
version: 1.13.6
resolution: "underscore@npm:1.13.6"
checksum: d5cedd14a9d0d91dd38c1ce6169e4455bb931f0aaf354108e47bd46d3f2da7464d49b2171a5cf786d61963204a42d01ea1332a903b7342ad428deaafaf70ec36
languageName: node
linkType: hard
"undici@npm:^5.1.0, undici@npm:^5.12.0, undici@npm:^5.19.1, undici@npm:^5.22.0, undici@npm:^5.8.0":
version: 5.22.1
resolution: "undici@npm:5.22.1"