feat(preview-service): re introduce preview service acceptance test (#5049)
* feat: basic structure for running tests * feat: added test to ci * feat: added server test (wip) * refactor: restuctured entrypoint * feat: added supertest * fix: missing deps * fix: test example ci * fix: updated default envs * feat: debug ci * feat: switch browser * fix: superadmin ci * feat: try another image * fix: try another image with node * fix: mr comments * fix: ci job * chore: workaround to push the image * chore: try with new base image * chore: retry * chore: retry * chore: retry * chore: retry * chore: retry * fix: test via debug * fix: envbar * chore: wrapped up changes, cleaned mr * chore: fix linter and skiped puppeteer download * fix: removed paralelism * fix: paralelism issues
This commit is contained in:
Daniel Gak Anagrov
committed by
GitHub
GitHub
parent
c7d97eb25c
commit
e6cd2ab441
@@ -198,7 +198,20 @@ jobs:
|
||||
test-preview-service:
|
||||
name: Preview service
|
||||
runs-on: blacksmith
|
||||
if: false # disabled as there is nothing to run
|
||||
container:
|
||||
image: ghcr.io/specklesystems/speckle-ubuntu-chromium:latest
|
||||
services:
|
||||
redis:
|
||||
image: redis:7.2.4
|
||||
options: >-
|
||||
--health-cmd "redis-cli ping"
|
||||
--health-interval 10s
|
||||
--health-timeout 5s
|
||||
--health-retries 5
|
||||
ports:
|
||||
- 6379:6379
|
||||
env:
|
||||
REDIS_URL: redis://redis:6379
|
||||
steps:
|
||||
- uses: actions/checkout@v4.2.2
|
||||
- uses: useblacksmith/setup-node@v5
|
||||
@@ -206,9 +219,19 @@ jobs:
|
||||
node-version: 22
|
||||
cache: yarn
|
||||
- name: Install dependencies
|
||||
run: YARN_ENABLE_HARDENED_MODE=0 PUPPETEER_SKIP_DOWNLOAD=true PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD=1 yarn --immutable
|
||||
run: YARN_ENABLE_HARDENED_MODE=0 PUPPETEER_SKIP_DOWNLOAD=true PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD=0 yarn --immutable
|
||||
- name: Build public packages
|
||||
run: yarn build:public
|
||||
- run: dbus-daemon --system &> /dev/null
|
||||
- run: cp .env.test-example .env.test
|
||||
working-directory: 'packages/preview-service'
|
||||
- run: yarn build:frontend
|
||||
working-directory: 'packages/preview-service'
|
||||
- run: yarn link:frontend
|
||||
working-directory: 'packages/preview-service'
|
||||
- name: Run tests
|
||||
run: yarn test:ci
|
||||
working-directory: 'packages/preview-service'
|
||||
|
||||
docker-build-postgres-container:
|
||||
runs-on: blacksmith
|
||||
|
||||
@@ -0,0 +1,11 @@
|
||||
PREVIEWS_HEADED='true'
|
||||
CHROMIUM_EXECUTABLE_PATH='/usr/bin/chromium'
|
||||
USER_DATA_DIR='/tmp/puppeteer'
|
||||
REDIS_URL='redis://localhost:6379'
|
||||
PORT='3099'
|
||||
PROMETHEUS_METRICS_PORT=''
|
||||
PREVIEWS_HEADED='false'
|
||||
|
||||
##########################################################
|
||||
# Test settings
|
||||
##########################################################
|
||||
@@ -1 +1,3 @@
|
||||
public
|
||||
tests/snapshots/diff.png
|
||||
tests/snapshots/result.png
|
||||
|
||||
@@ -19,7 +19,8 @@
|
||||
"link:frontend": "yarn build:frontend && rimraf ./public && ln -s ../preview-frontend/dist ./public",
|
||||
"dev": "tsx --env-file=.env --watch src/main.ts",
|
||||
"publishTask": "tsx --env-file=.env scripts/publishTask.ts",
|
||||
"test": "echo 'no tests configured'",
|
||||
"test": "NODE_ENV=test vitest run --sequence.shuffle",
|
||||
"test:ci": "NODE_ENV=test LOG_LEVEL=silent LOG_PRETTY=true vitest run --sequence.shuffle",
|
||||
"lint": "yarn lint:tsc && yarn lint:eslint",
|
||||
"lint:ci": "yarn lint:tsc",
|
||||
"lint:tsc": "tsc --noEmit",
|
||||
@@ -46,14 +47,19 @@
|
||||
"@swc/core": "^1.9.3",
|
||||
"@types/express": "^4.17.13",
|
||||
"@types/node": "^18.19.38",
|
||||
"@types/pngjs": "^6.0.5",
|
||||
"eslint": "^9.4.0",
|
||||
"eslint-config-prettier": "^9.1.0",
|
||||
"eslint-plugin-vitest": "^0.5.4",
|
||||
"pixelmatch": "^7.1.0",
|
||||
"pngjs": "^7.0.0",
|
||||
"prettier": "^2.5.1",
|
||||
"rimraf": "^6.0.1",
|
||||
"supertest": "^7.1.3",
|
||||
"ts-node": "^10.9.2",
|
||||
"tsx": "^4.19.2",
|
||||
"typescript": "^4.6.4",
|
||||
"typescript-eslint": "^7.12.0"
|
||||
"typescript-eslint": "^7.12.0",
|
||||
"vitest": "^1.6.0"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,8 +2,6 @@ import generateAliasesResolver from 'esm-module-alias'
|
||||
import path from 'node:path'
|
||||
import fs from 'node:fs'
|
||||
import { fileURLToPath } from 'url'
|
||||
import dotenv from 'dotenv'
|
||||
dotenv.config()
|
||||
|
||||
/**
|
||||
* Singleton module for src root and package root directory resolution
|
||||
|
||||
@@ -1,3 +1,9 @@
|
||||
import dotenv from 'dotenv'
|
||||
|
||||
export const isTestEnvironment = () => process.env['NODE_ENV'] === 'test'
|
||||
|
||||
dotenv.config(isTestEnvironment() ? { path: '.env.test' } : {})
|
||||
|
||||
import { z } from 'zod'
|
||||
import { parseEnv } from 'znv'
|
||||
|
||||
|
||||
@@ -1,232 +1,15 @@
|
||||
import express from 'express'
|
||||
import puppeteer, { Browser } from 'puppeteer'
|
||||
import { createTerminus } from '@godaddy/terminus'
|
||||
import type { Logger } from 'pino'
|
||||
import type Bull from 'bull'
|
||||
|
||||
import { JobPayload, PreviewResultPayload } from '@speckle/shared/workers/previews'
|
||||
import { AppState } from '@speckle/shared/workers'
|
||||
import {
|
||||
REDIS_URL,
|
||||
HOST,
|
||||
PORT,
|
||||
CHROMIUM_EXECUTABLE_PATH,
|
||||
PREVIEWS_HEADED,
|
||||
USER_DATA_DIR,
|
||||
PREVIEW_TIMEOUT,
|
||||
GPU_ENABLED
|
||||
} from '@/config.js'
|
||||
import { logger } from '@/logging.js'
|
||||
import { jobProcessor } from '@/jobProcessor.js'
|
||||
import { isTestEnvironment } from '@/config.js'
|
||||
import { initMetrics, initPrometheusRegistry } from '@/metrics.js'
|
||||
import { ensureError } from '@speckle/shared'
|
||||
import { initializeQueue } from '@speckle/shared/queue'
|
||||
import { isRedisReady } from '@speckle/shared/redis'
|
||||
import { buildServer, initServer } from './server'
|
||||
|
||||
const app = express()
|
||||
const host = HOST
|
||||
const port = PORT
|
||||
|
||||
const JobQueueName = 'preview-service-jobs'
|
||||
|
||||
let appState: AppState = AppState.STARTING
|
||||
|
||||
// serve the preview-frontend
|
||||
app.use(express.static('public'))
|
||||
await initMetrics({ app, registry: initPrometheusRegistry() })
|
||||
let jobQueue: Bull.Queue<JobPayload> | undefined = undefined
|
||||
|
||||
// store this callback, so on shutdown we can error the job
|
||||
let currentJob: { logger: Logger; done: Bull.DoneCallback } | undefined = undefined
|
||||
if (!isTestEnvironment()) await initMetrics({ app, registry: initPrometheusRegistry() })
|
||||
|
||||
// browser is a global variable, so we can handle the shutdown of the browser
|
||||
// in the beforeShutdown function. We need to stop processing jobs before we
|
||||
// can close the browser
|
||||
let browser: Browser | undefined = undefined
|
||||
const server = buildServer({ app })
|
||||
|
||||
const server = app.listen(port, host, async () => {
|
||||
logger.info({ port }, '📡 Started Preview Service server, listening on {port}')
|
||||
appState = AppState.RUNNING
|
||||
|
||||
const gpuArgs = ['--use-gl=angle', '--use-angle=gl-egl']
|
||||
|
||||
const launchBrowser = async (): Promise<Browser> => {
|
||||
const launchArguments = [
|
||||
'--no-sandbox',
|
||||
'--disable-setuid-sandbox',
|
||||
'--disable-dev-shm-usage',
|
||||
'--disable-session-crashed-bubble',
|
||||
...(GPU_ENABLED ? gpuArgs : [])
|
||||
]
|
||||
logger.debug(
|
||||
`Starting browser, located at "${CHROMIUM_EXECUTABLE_PATH}", with the following arguments: ${JSON.stringify(
|
||||
launchArguments
|
||||
)}`
|
||||
)
|
||||
return await puppeteer.launch({
|
||||
headless: !PREVIEWS_HEADED,
|
||||
executablePath: CHROMIUM_EXECUTABLE_PATH,
|
||||
userDataDir: USER_DATA_DIR,
|
||||
// slowMo: 3000, // Use for debugging during development
|
||||
// we trust the web content that is running, so can disable the sandbox
|
||||
// disabling the sandbox allows us to run the docker image without linux kernel privileges
|
||||
args: launchArguments,
|
||||
protocolTimeout: PREVIEW_TIMEOUT,
|
||||
// handle closing of the browser by the preview-service, not puppeteer
|
||||
// this is important for the preview-service to be able to shut down gracefully,
|
||||
// otherwise we end up in race condition where puppeteer closes before preview-service
|
||||
handleSIGHUP: false,
|
||||
handleSIGINT: false,
|
||||
handleSIGTERM: false
|
||||
})
|
||||
}
|
||||
|
||||
try {
|
||||
jobQueue = await initializeQueue<JobPayload>({
|
||||
queueName: JobQueueName,
|
||||
redisUrl: REDIS_URL
|
||||
})
|
||||
} catch (e) {
|
||||
const err = ensureError(e, 'Unknown error creating job queue')
|
||||
logger.error({ err }, 'Error creating job queue')
|
||||
|
||||
// the callback to server.listen has failed, so we need to exit the process and not just return
|
||||
await beforeShutdown() // handle the shutdown gracefully
|
||||
await onShutdown()
|
||||
process.exit(1)
|
||||
}
|
||||
|
||||
logger.debug(`Starting processing of "${JobQueueName}" message queue`)
|
||||
|
||||
// nothing after this line is getting called, this blocks
|
||||
await jobQueue.process(async (payload, done) => {
|
||||
let encounteredError = false
|
||||
let jobLogger = logger.child({
|
||||
payloadId: payload.id,
|
||||
jobPriorAttemptsMade: payload.attemptsMade
|
||||
})
|
||||
|
||||
if (browser) {
|
||||
const message = 'Tried to start job but Browser is already open.'
|
||||
done(new Error(message))
|
||||
throw new Error(message)
|
||||
}
|
||||
|
||||
try {
|
||||
currentJob = { done, logger: jobLogger }
|
||||
const job = payload.data
|
||||
jobLogger = jobLogger.child({
|
||||
jobId: job.jobId,
|
||||
serverUrl: job.url
|
||||
})
|
||||
const resultsQueue = await initializeQueue<PreviewResultPayload>({
|
||||
queueName: job.responseQueue,
|
||||
redisUrl: REDIS_URL
|
||||
})
|
||||
|
||||
browser = await launchBrowser()
|
||||
const result = await jobProcessor({
|
||||
logger: jobLogger,
|
||||
browser,
|
||||
job,
|
||||
port: PORT,
|
||||
timeout: PREVIEW_TIMEOUT,
|
||||
getAppState: () => appState
|
||||
})
|
||||
|
||||
// with removeOnComplete, the job response potentially containing a large images,
|
||||
// is cleared from the response queue
|
||||
await resultsQueue.add(result, { removeOnComplete: true })
|
||||
} catch (err) {
|
||||
if (appState === AppState.SHUTTINGDOWN) {
|
||||
// likely that the job was cancelled due to the service shutting down
|
||||
jobLogger.warn({ err }, 'Processing job {jobId} failed')
|
||||
} else {
|
||||
jobLogger.error({ err }, 'Processing job {jobId} failed')
|
||||
}
|
||||
if (err instanceof Error) {
|
||||
encounteredError = true
|
||||
done(err)
|
||||
} else {
|
||||
throw err
|
||||
}
|
||||
} finally {
|
||||
if (browser) await browser.close()
|
||||
browser = undefined
|
||||
if (!encounteredError) done()
|
||||
currentJob = undefined
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
const beforeShutdown = async () => {
|
||||
logger.info('🛑 Beginning shut down, pausing all jobs')
|
||||
appState = AppState.SHUTTINGDOWN
|
||||
// stop accepting new jobs and kill any running jobs
|
||||
if (jobQueue) {
|
||||
await jobQueue.pause(
|
||||
true, // just pausing this local worker of the queue
|
||||
true // do not wait for active jobs to finish
|
||||
)
|
||||
}
|
||||
|
||||
if (currentJob) {
|
||||
currentJob.logger.warn('Cancelling job due to preview-service shutdown')
|
||||
currentJob.done(new Error('Job cancelled due to preview-service shutdown'))
|
||||
}
|
||||
if (browser) {
|
||||
// preview-service is responsible for closing the browser
|
||||
// to allow us to stop listening for new jobs and properly respond to any
|
||||
// current job before we kill the browser
|
||||
logger.info('Closing browser')
|
||||
await browser.close()
|
||||
browser = undefined
|
||||
}
|
||||
// no need to close the job queue and redis client, when the process exits they will be closed automatically
|
||||
}
|
||||
|
||||
const onShutdown = async () => {
|
||||
logger.info('👋 Completed shut down, now exiting')
|
||||
}
|
||||
|
||||
createTerminus(server, {
|
||||
healthChecks: {
|
||||
'/liveness': () => Promise.resolve('ok'),
|
||||
'/readiness': async (args: { state: { isShuttingDown: boolean } }) => {
|
||||
const { isShuttingDown } = args.state
|
||||
if (isShuttingDown) {
|
||||
return Promise.reject(new Error('Preview service is shutting down'))
|
||||
}
|
||||
|
||||
if (!jobQueue) {
|
||||
return Promise.reject(new Error('Job queue is not initialized'))
|
||||
}
|
||||
|
||||
try {
|
||||
await isRedisReady(jobQueue.client)
|
||||
} catch (e) {
|
||||
return Promise.reject(
|
||||
ensureError(e, 'Unknown error when checking Redis client')
|
||||
)
|
||||
}
|
||||
const isReady = await jobQueue.isReady()
|
||||
if (!isReady)
|
||||
return Promise.reject(
|
||||
new Error(
|
||||
'Preview service is not ready. Redis or Bull is not either reachable or ready.'
|
||||
)
|
||||
)
|
||||
|
||||
return Promise.resolve('ok')
|
||||
}
|
||||
},
|
||||
beforeShutdown,
|
||||
onShutdown,
|
||||
logger: (msg, err) => {
|
||||
if (err) {
|
||||
logger.error({ err }, msg)
|
||||
return
|
||||
}
|
||||
logger.info(msg)
|
||||
}
|
||||
})
|
||||
initServer(server)
|
||||
|
||||
@@ -0,0 +1,229 @@
|
||||
import puppeteer, { Browser } from 'puppeteer'
|
||||
import { JobPayload, PreviewResultPayload } from '@speckle/shared/workers/previews'
|
||||
import { AppState } from '@speckle/shared/workers'
|
||||
import type Bull from 'bull'
|
||||
import {
|
||||
REDIS_URL,
|
||||
HOST,
|
||||
PORT,
|
||||
CHROMIUM_EXECUTABLE_PATH,
|
||||
PREVIEWS_HEADED,
|
||||
USER_DATA_DIR,
|
||||
PREVIEW_TIMEOUT,
|
||||
GPU_ENABLED
|
||||
} from '@/config.js'
|
||||
|
||||
import { jobProcessor } from '@/jobProcessor.js'
|
||||
import { initializeQueue } from '@speckle/shared/queue'
|
||||
import { Express } from 'express'
|
||||
import { logger } from '@/logging.js'
|
||||
import { Logger } from 'pino'
|
||||
const JobQueueName = 'preview-service-jobs'
|
||||
import { ensureError } from '@speckle/shared'
|
||||
import { createTerminus } from '@godaddy/terminus'
|
||||
import { isRedisReady } from '@speckle/shared/redis'
|
||||
import { Server } from 'http'
|
||||
|
||||
let appState: AppState = AppState.STARTING
|
||||
|
||||
let jobQueue: Bull.Queue<JobPayload> | undefined = undefined
|
||||
|
||||
// store this callback, so on shutdown we can error the job
|
||||
let currentJob: { logger: Logger; done: Bull.DoneCallback } | undefined = undefined
|
||||
|
||||
// browser is a global variable, so we can handle the shutdown of the browser
|
||||
// in the beforeShutdown function. We need to stop processing jobs before we
|
||||
// can close the browser
|
||||
let browser: Browser | undefined = undefined
|
||||
|
||||
export const buildServer = ({
|
||||
port,
|
||||
host,
|
||||
app
|
||||
}: Partial<{ port: number; host: string }> & { app: Express }) =>
|
||||
app.listen(port || PORT, host || HOST, async () => {
|
||||
logger.info({ port }, '📡 Started Preview Service server, listening on {port}')
|
||||
appState = AppState.RUNNING
|
||||
|
||||
const gpuArgs = ['--use-gl=angle', '--use-angle=gl-egl']
|
||||
|
||||
const launchBrowser = async (): Promise<Browser> => {
|
||||
const launchArguments = [
|
||||
'--no-sandbox',
|
||||
'--disable-setuid-sandbox',
|
||||
'--disable-dev-shm-usage',
|
||||
'--disable-session-crashed-bubble',
|
||||
...(GPU_ENABLED ? gpuArgs : [])
|
||||
]
|
||||
logger.debug(
|
||||
`Starting browser, located at "${CHROMIUM_EXECUTABLE_PATH}", with the following arguments: ${JSON.stringify(
|
||||
launchArguments
|
||||
)}`
|
||||
)
|
||||
return await puppeteer.launch({
|
||||
headless: !PREVIEWS_HEADED,
|
||||
executablePath: CHROMIUM_EXECUTABLE_PATH,
|
||||
userDataDir: USER_DATA_DIR,
|
||||
// slowMo: 3000, // Use for debugging during development
|
||||
// we trust the web content that is running, so can disable the sandbox
|
||||
// disabling the sandbox allows us to run the docker image without linux kernel privileges
|
||||
args: launchArguments,
|
||||
protocolTimeout: PREVIEW_TIMEOUT,
|
||||
// handle closing of the browser by the preview-service, not puppeteer
|
||||
// this is important for the preview-service to be able to shut down gracefully,
|
||||
// otherwise we end up in race condition where puppeteer closes before preview-service
|
||||
handleSIGHUP: false,
|
||||
handleSIGINT: false,
|
||||
handleSIGTERM: false
|
||||
})
|
||||
}
|
||||
|
||||
try {
|
||||
jobQueue = await initializeQueue<JobPayload>({
|
||||
queueName: JobQueueName,
|
||||
redisUrl: REDIS_URL
|
||||
})
|
||||
} catch (e) {
|
||||
const err = ensureError(e, 'Unknown error creating job queue')
|
||||
logger.error({ err }, 'Error creating job queue')
|
||||
|
||||
// the callback to server.listen has failed, so we need to exit the process and not just return
|
||||
await beforeShutdown() // handle the shutdown gracefully
|
||||
await onShutdown()
|
||||
process.exit(1)
|
||||
}
|
||||
logger.debug(`Starting processing of "${JobQueueName}" message queue`)
|
||||
|
||||
// nothing after this line is getting called, this blocks
|
||||
await jobQueue.process(async (payload, done) => {
|
||||
let encounteredError = false
|
||||
let jobLogger = logger.child({
|
||||
payloadId: payload.id,
|
||||
jobPriorAttemptsMade: payload.attemptsMade
|
||||
})
|
||||
|
||||
if (browser) {
|
||||
const message = 'Tried to start job but Browser is already open.'
|
||||
done(new Error(message))
|
||||
throw new Error(message)
|
||||
}
|
||||
|
||||
try {
|
||||
currentJob = { done, logger: jobLogger }
|
||||
const job = payload.data
|
||||
jobLogger = jobLogger.child({
|
||||
jobId: job.jobId,
|
||||
serverUrl: job.url
|
||||
})
|
||||
const resultsQueue = await initializeQueue<PreviewResultPayload>({
|
||||
queueName: job.responseQueue,
|
||||
redisUrl: REDIS_URL
|
||||
})
|
||||
|
||||
browser = await launchBrowser()
|
||||
const result = await jobProcessor({
|
||||
logger: jobLogger,
|
||||
browser,
|
||||
job,
|
||||
port: PORT,
|
||||
timeout: PREVIEW_TIMEOUT,
|
||||
getAppState: () => appState
|
||||
})
|
||||
|
||||
// with removeOnComplete, the job response potentially containing a large images,
|
||||
// is cleared from the response queue
|
||||
await resultsQueue.add(result, { removeOnComplete: true })
|
||||
} catch (err) {
|
||||
if (appState === AppState.SHUTTINGDOWN) {
|
||||
// likely that the job was cancelled due to the service shutting down
|
||||
jobLogger.warn({ err }, 'Processing job {jobId} failed')
|
||||
} else {
|
||||
jobLogger.error({ err }, 'Processing job {jobId} failed')
|
||||
}
|
||||
if (err instanceof Error) {
|
||||
encounteredError = true
|
||||
done(err)
|
||||
} else {
|
||||
throw err
|
||||
}
|
||||
} finally {
|
||||
if (browser) await browser.close()
|
||||
browser = undefined
|
||||
if (!encounteredError) done()
|
||||
currentJob = undefined
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
export const initServer = (server: Server) =>
|
||||
createTerminus(server, {
|
||||
healthChecks: {
|
||||
'/liveness': () => Promise.resolve('ok'),
|
||||
'/readiness': isReady
|
||||
},
|
||||
beforeShutdown,
|
||||
onShutdown,
|
||||
logger: (msg, err) => {
|
||||
if (err) {
|
||||
logger.error({ err }, msg)
|
||||
return
|
||||
}
|
||||
logger.info(msg)
|
||||
}
|
||||
})
|
||||
|
||||
const beforeShutdown = async () => {
|
||||
logger.info('🛑 Beginning shut down, pausing all jobs')
|
||||
appState = AppState.SHUTTINGDOWN
|
||||
// stop accepting new jobs and kill any running jobs
|
||||
if (jobQueue) {
|
||||
await jobQueue.pause(
|
||||
true, // just pausing this local worker of the queue
|
||||
true // do not wait for active jobs to finish
|
||||
)
|
||||
}
|
||||
|
||||
if (currentJob) {
|
||||
currentJob.logger.warn('Cancelling job due to preview-service shutdown')
|
||||
currentJob.done(new Error('Job cancelled due to preview-service shutdown'))
|
||||
}
|
||||
if (browser) {
|
||||
// preview-service is responsible for closing the browser
|
||||
// to allow us to stop listening for new jobs and properly respond to any
|
||||
// current job before we kill the browser
|
||||
logger.info('Closing browser')
|
||||
await browser.close()
|
||||
browser = undefined
|
||||
}
|
||||
// no need to close the job queue and redis client, when the process exits they will be closed automatically
|
||||
}
|
||||
|
||||
const onShutdown = async () => {
|
||||
logger.info('👋 Completed shut down, now exiting')
|
||||
}
|
||||
|
||||
const isReady = async (args: { state: { isShuttingDown: boolean } }) => {
|
||||
const { isShuttingDown } = args.state
|
||||
if (isShuttingDown) {
|
||||
return Promise.reject(new Error('Preview service is shutting down'))
|
||||
}
|
||||
|
||||
if (!jobQueue) {
|
||||
return Promise.reject(new Error('Job queue is not initialized'))
|
||||
}
|
||||
|
||||
try {
|
||||
await isRedisReady(jobQueue.client)
|
||||
} catch (e) {
|
||||
return Promise.reject(ensureError(e, 'Unknown error when checking Redis client'))
|
||||
}
|
||||
const isReady = await jobQueue.isReady()
|
||||
if (!isReady)
|
||||
return Promise.reject(
|
||||
new Error(
|
||||
'Preview service is not ready. Redis or Bull is not either reachable or ready.'
|
||||
)
|
||||
)
|
||||
|
||||
return Promise.resolve('ok')
|
||||
}
|
||||
@@ -0,0 +1,141 @@
|
||||
import { describe, it, beforeAll, afterAll, expect } from 'vitest'
|
||||
import express from 'express'
|
||||
import { buildServer } from 'src/server'
|
||||
import { Server } from 'http'
|
||||
import { initializeQueue } from '@speckle/shared/queue'
|
||||
import { REDIS_URL } from '@/config.js'
|
||||
import Bull from 'bull'
|
||||
import { JobPayload } from '@speckle/shared/workers/previews'
|
||||
import { randomUUID } from 'crypto'
|
||||
import supertest, { SuperTest, Test } from 'supertest'
|
||||
import { TIME_MS } from '@speckle/shared'
|
||||
import fs from 'fs'
|
||||
import pixelmatch from 'pixelmatch'
|
||||
import { PNG } from 'pngjs'
|
||||
import path from 'path'
|
||||
|
||||
const BASE_IMAGE = path.resolve(__dirname, 'snapshots/base.png')
|
||||
const TEST_RESULT = path.resolve(__dirname, 'snapshots/result.png')
|
||||
const DIFF = path.resolve(__dirname, 'snapshots/diff.png')
|
||||
|
||||
describe('preview-service', () => {
|
||||
let server: Server
|
||||
let request: SuperTest<Test>
|
||||
let jobQueue: Bull.Queue<JobPayload>
|
||||
let responseQueue: Bull.Queue<{
|
||||
jobId: string
|
||||
status: string
|
||||
result: {
|
||||
screenshots: object
|
||||
}
|
||||
}>
|
||||
|
||||
const testId = randomUUID()
|
||||
const JOB_QUEUE = 'preview-service-jobs'
|
||||
const RESPONSE_QUEUE = 'preview-service-jobs-test-queue-' + testId
|
||||
|
||||
const sleep = async (ms: number) => {
|
||||
await new Promise((resolve) => setTimeout(resolve, ms))
|
||||
}
|
||||
|
||||
beforeAll(async () => {
|
||||
const app = express()
|
||||
app.use(express.static('public'))
|
||||
request = supertest(app)
|
||||
server = buildServer({ app })
|
||||
|
||||
jobQueue = await initializeQueue({
|
||||
queueName: JOB_QUEUE,
|
||||
redisUrl: REDIS_URL
|
||||
})
|
||||
|
||||
responseQueue = await initializeQueue({
|
||||
queueName: RESPONSE_QUEUE,
|
||||
redisUrl: REDIS_URL
|
||||
})
|
||||
|
||||
// delete existing images
|
||||
fs.rmSync(TEST_RESULT, { recursive: false, force: true })
|
||||
fs.rmSync(DIFF, { recursive: false, force: true })
|
||||
|
||||
// TODO: remove this head start
|
||||
// only awaiting for the server to start does not work
|
||||
// we should await the start of the job processing
|
||||
await sleep(6 * TIME_MS.second)
|
||||
})
|
||||
|
||||
afterAll(async () => {
|
||||
server.close()
|
||||
await jobQueue.close()
|
||||
await responseQueue.close()
|
||||
})
|
||||
|
||||
it('inits a server', async () => {
|
||||
expect(server).to.be.instanceOf(Server)
|
||||
})
|
||||
|
||||
it('hits the server', async () => {
|
||||
const response = await request.get('/')
|
||||
|
||||
expect(response.status).to.equal(200)
|
||||
})
|
||||
|
||||
it('process a rendering task providing back the image', async () => {
|
||||
const ID = 'test-job' + testId
|
||||
|
||||
await jobQueue.add({
|
||||
url: 'https://latest.speckle.systems/projects/8b94a55ee5/models/7f98c5b62e',
|
||||
token: '',
|
||||
jobId: ID,
|
||||
responseQueue: RESPONSE_QUEUE
|
||||
})
|
||||
|
||||
let jobs = null
|
||||
while (!jobs || !jobs.length) {
|
||||
// can run until the test suite times out
|
||||
jobs = await responseQueue.getJobs(['waiting'])
|
||||
|
||||
await sleep(1 * TIME_MS.second)
|
||||
}
|
||||
|
||||
expect(jobs).to.have.lengthOf(1)
|
||||
const [job] = jobs
|
||||
expect(job.data.jobId).to.equal(ID)
|
||||
expect(job.data.status).to.equal('success')
|
||||
expect(job.data.result).to.be.an('object')
|
||||
expect(job.data.result.screenshots).toBeDefined()
|
||||
|
||||
// write the image to a result file, for debugging
|
||||
|
||||
const image =
|
||||
'0' in job.data.result.screenshots
|
||||
? (job.data.result.screenshots['0'] as string)
|
||||
: null
|
||||
|
||||
if (!image) expect.fail('No image found')
|
||||
|
||||
const clean = Buffer.from(image.replace(/^data:image\/png;base64,/, ''), 'base64')
|
||||
|
||||
fs.writeFileSync(TEST_RESULT, clean)
|
||||
|
||||
// test max diff
|
||||
|
||||
const base = PNG.sync.read(fs.readFileSync(BASE_IMAGE))
|
||||
const result = PNG.sync.read(fs.readFileSync(TEST_RESULT))
|
||||
const diff = new PNG({ width: base.width, height: base.height })
|
||||
const totalPixels = base.width * base.height
|
||||
|
||||
const diffPixels = pixelmatch(
|
||||
base.data,
|
||||
result.data,
|
||||
diff.data,
|
||||
base.width,
|
||||
base.height,
|
||||
{ threshold: 0.1 }
|
||||
)
|
||||
|
||||
fs.writeFileSync(DIFF, PNG.sync.write(diff))
|
||||
const diffPercentage = Number(((diffPixels / totalPixels) * 100).toFixed(2))
|
||||
expect(diffPercentage).to.be.lessThan(10)
|
||||
})
|
||||
})
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 86 KiB |
@@ -0,0 +1,20 @@
|
||||
import { TIME_MS } from '@speckle/shared'
|
||||
import path from 'path'
|
||||
import { configDefaults, defineConfig } from 'vitest/config'
|
||||
|
||||
export default defineConfig({
|
||||
test: {
|
||||
exclude: [...configDefaults.exclude],
|
||||
// reporters: ['verbose', 'hanging-process'] //uncomment to debug hanging processes etc.
|
||||
sequence: {
|
||||
shuffle: true,
|
||||
concurrent: true
|
||||
},
|
||||
testTimeout: 2 * TIME_MS.minute
|
||||
},
|
||||
resolve: {
|
||||
alias: {
|
||||
'@': path.resolve(__dirname, './src')
|
||||
}
|
||||
}
|
||||
})
|
||||
@@ -0,0 +1,30 @@
|
||||
FROM node:20-slim
|
||||
|
||||
# hadolint ignore=DL3008
|
||||
RUN apt-get update && \
|
||||
apt-get install -y \
|
||||
chromium \
|
||||
fonts-liberation \
|
||||
libappindicator3-1 \
|
||||
libasound2 \
|
||||
libatk-bridge2.0-0 \
|
||||
libatk1.0-0 \
|
||||
libcups2 \
|
||||
libdbus-1-3 \
|
||||
libgdk-pixbuf2.0-0 \
|
||||
libnspr4 \
|
||||
libnss3 \
|
||||
libxcomposite1 \
|
||||
libxdamage1 \
|
||||
libxrandr2 \
|
||||
xdg-utils \
|
||||
wget \
|
||||
ca-certificates \
|
||||
--no-install-recommends && \
|
||||
apt-get clean && \
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
|
||||
ENV PUPPETEER_EXECUTABLE_PATH=/usr/bin/chromium
|
||||
|
||||
# Create app directory
|
||||
WORKDIR /usr/src/app
|
||||
@@ -12731,6 +12731,13 @@ __metadata:
|
||||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
"@noble/hashes@npm:^1.1.5":
|
||||
version: 1.8.0
|
||||
resolution: "@noble/hashes@npm:1.8.0"
|
||||
checksum: 10/474b7f56bc6fb2d5b3a42132561e221b0ea4f91e590f4655312ca13667840896b34195e2b53b7f097ec080a1fdd3b58d902c2a8d0fbdf51d2e238b53808a177e
|
||||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
"@nodelib/fs.scandir@npm:2.1.5":
|
||||
version: 2.1.5
|
||||
resolution: "@nodelib/fs.scandir@npm:2.1.5"
|
||||
@@ -13586,6 +13593,15 @@ __metadata:
|
||||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
"@paralleldrive/cuid2@npm:^2.2.2":
|
||||
version: 2.2.2
|
||||
resolution: "@paralleldrive/cuid2@npm:2.2.2"
|
||||
dependencies:
|
||||
"@noble/hashes": "npm:^1.1.5"
|
||||
checksum: 10/40ee269d6e47b4fed7706a2e4da7c27c3c668ebc969110d6d112277b6b16a67cce0503b53b9943f2c55035a72d225f77ea5541e03396d6429eec9252137a53b7
|
||||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
"@parcel/watcher-android-arm64@npm:2.4.1":
|
||||
version: 2.4.1
|
||||
resolution: "@parcel/watcher-android-arm64@npm:2.4.1"
|
||||
@@ -16191,6 +16207,7 @@ __metadata:
|
||||
"@swc/core": "npm:^1.9.3"
|
||||
"@types/express": "npm:^4.17.13"
|
||||
"@types/node": "npm:^18.19.38"
|
||||
"@types/pngjs": "npm:^6.0.5"
|
||||
bull: "npm:^4.16.4"
|
||||
dotenv: "npm:^16.4.7"
|
||||
eslint: "npm:^9.4.0"
|
||||
@@ -16201,14 +16218,18 @@ __metadata:
|
||||
pino: "npm:^8.7.0"
|
||||
pino-http: "npm:^8.6.1"
|
||||
pino-pretty: "npm:^9.1.1"
|
||||
pixelmatch: "npm:^7.1.0"
|
||||
pngjs: "npm:^7.0.0"
|
||||
prettier: "npm:^2.5.1"
|
||||
prom-client: "npm:^14.0.1"
|
||||
puppeteer: "npm:^23.9.0"
|
||||
rimraf: "npm:^6.0.1"
|
||||
supertest: "npm:^7.1.3"
|
||||
ts-node: "npm:^10.9.2"
|
||||
tsx: "npm:^4.19.2"
|
||||
typescript: "npm:^4.6.4"
|
||||
typescript-eslint: "npm:^7.12.0"
|
||||
vitest: "npm:^1.6.0"
|
||||
znv: "npm:^0.4.0"
|
||||
zod: "npm:^3.23.8"
|
||||
languageName: unknown
|
||||
@@ -19594,6 +19615,15 @@ __metadata:
|
||||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
"@types/pngjs@npm:^6.0.5":
|
||||
version: 6.0.5
|
||||
resolution: "@types/pngjs@npm:6.0.5"
|
||||
dependencies:
|
||||
"@types/node": "npm:*"
|
||||
checksum: 10/132fce25817d47a784ed48aa678332521b0f7e6edbaa76f3fa4e9ca1228078788ae712f99ad4d1a324d9ba0b14829958677eabf3ebef1fb6e120816f433f0cd8
|
||||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
"@types/polylabel@npm:1.1.3":
|
||||
version: 1.1.3
|
||||
resolution: "@types/polylabel@npm:1.1.3"
|
||||
@@ -22255,7 +22285,7 @@ __metadata:
|
||||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
"asap@npm:~2.0.3":
|
||||
"asap@npm:^2.0.0, asap@npm:~2.0.3":
|
||||
version: 2.0.6
|
||||
resolution: "asap@npm:2.0.6"
|
||||
checksum: 10/b244c0458c571945e4b3be0b14eb001bea5596f9868cc50cc711dc03d58a7e953517d3f0dad81ccde3ff37d1f074701fa76a6f07d41aaa992d7204a37b915dda
|
||||
@@ -24874,6 +24904,13 @@ __metadata:
|
||||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
"component-emitter@npm:^1.3.0":
|
||||
version: 1.3.1
|
||||
resolution: "component-emitter@npm:1.3.1"
|
||||
checksum: 10/94550aa462c7bd5a61c1bc480e28554aa306066930152d1b1844a0dd3845d4e5db7e261ddec62ae184913b3e59b55a2ad84093b9d3596a8f17c341514d6c483d
|
||||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
"compress-commons@npm:^6.0.2":
|
||||
version: 6.0.2
|
||||
resolution: "compress-commons@npm:6.0.2"
|
||||
@@ -25223,7 +25260,7 @@ __metadata:
|
||||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
"cookiejar@npm:^2.1.0, cookiejar@npm:^2.1.1":
|
||||
"cookiejar@npm:^2.1.0, cookiejar@npm:^2.1.1, cookiejar@npm:^2.1.4":
|
||||
version: 2.1.4
|
||||
resolution: "cookiejar@npm:2.1.4"
|
||||
checksum: 10/4a184f5a0591df8b07d22a43ea5d020eacb4572c383e853a33361a99710437eaa0971716c688684075bbf695b484f5872e9e3f562382e46858716cb7fc8ce3f4
|
||||
@@ -26725,6 +26762,16 @@ __metadata:
|
||||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
"dezalgo@npm:^1.0.4":
|
||||
version: 1.0.4
|
||||
resolution: "dezalgo@npm:1.0.4"
|
||||
dependencies:
|
||||
asap: "npm:^2.0.0"
|
||||
wrappy: "npm:1"
|
||||
checksum: 10/895389c6aead740d2ab5da4d3466d20fa30f738010a4d3f4dcccc9fc645ca31c9d10b7e1804ae489b1eb02c7986f9f1f34ba132d409b043082a86d9a4e745624
|
||||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
"didyoumean@npm:^1.2.2":
|
||||
version: 1.2.2
|
||||
resolution: "didyoumean@npm:1.2.2"
|
||||
@@ -30197,6 +30244,17 @@ __metadata:
|
||||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
"formidable@npm:^3.5.4":
|
||||
version: 3.5.4
|
||||
resolution: "formidable@npm:3.5.4"
|
||||
dependencies:
|
||||
"@paralleldrive/cuid2": "npm:^2.2.2"
|
||||
dezalgo: "npm:^1.0.4"
|
||||
once: "npm:^1.4.0"
|
||||
checksum: 10/4645e6ce3d8bbefd3dd873dcd6211362da3bf8a04c8426d7f454c238be0142975f02e5bdbc792fdbd2be493fdcf5442fe01d9a246bd8c3fd8e779738290cc630
|
||||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
"forwarded@npm:0.2.0":
|
||||
version: 0.2.0
|
||||
resolution: "forwarded@npm:0.2.0"
|
||||
@@ -36614,7 +36672,7 @@ __metadata:
|
||||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
"mime@npm:^2.4.6":
|
||||
"mime@npm:2.6.0, mime@npm:^2.4.6":
|
||||
version: 2.6.0
|
||||
resolution: "mime@npm:2.6.0"
|
||||
bin:
|
||||
@@ -40669,6 +40727,17 @@ __metadata:
|
||||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
"pixelmatch@npm:^7.1.0":
|
||||
version: 7.1.0
|
||||
resolution: "pixelmatch@npm:7.1.0"
|
||||
dependencies:
|
||||
pngjs: "npm:^7.0.0"
|
||||
bin:
|
||||
pixelmatch: bin/pixelmatch
|
||||
checksum: 10/57a122196318ea8ce74e8759b1b7b94b9f9627b495cd79e50a49d470dc23b6c679e89c38660d0f7e8f959eac3b279c55b728e52d02c276dc51505f06eaba1141
|
||||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
"pkg-dir@npm:^3.0.0":
|
||||
version: 3.0.0
|
||||
resolution: "pkg-dir@npm:3.0.0"
|
||||
@@ -40807,6 +40876,13 @@ __metadata:
|
||||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
"pngjs@npm:^7.0.0":
|
||||
version: 7.0.0
|
||||
resolution: "pngjs@npm:7.0.0"
|
||||
checksum: 10/e843ebbb0df092ee0f3a3e7dbd91ff87a239a4e4c4198fff202916bfb33b67622f4b83b3c29f3ccae94fcb97180c289df06068624554f61686fe6b9a4811f7db
|
||||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
"polished@npm:^4.2.2":
|
||||
version: 4.2.2
|
||||
resolution: "polished@npm:4.2.2"
|
||||
@@ -46308,6 +46384,23 @@ __metadata:
|
||||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
"superagent@npm:^10.2.2":
|
||||
version: 10.2.2
|
||||
resolution: "superagent@npm:10.2.2"
|
||||
dependencies:
|
||||
component-emitter: "npm:^1.3.0"
|
||||
cookiejar: "npm:^2.1.4"
|
||||
debug: "npm:^4.3.4"
|
||||
fast-safe-stringify: "npm:^2.1.1"
|
||||
form-data: "npm:^4.0.0"
|
||||
formidable: "npm:^3.5.4"
|
||||
methods: "npm:^1.1.2"
|
||||
mime: "npm:2.6.0"
|
||||
qs: "npm:^6.11.0"
|
||||
checksum: 10/e89ae49163df0db50e6a77316a7304a16640df11a8d2219bef11e69f59c74e54c16670ec250c2ab59f06887f71bb5d6e5933f735b931cbdbe34ba490d78b5d70
|
||||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
"superagent@npm:^3.7.0, superagent@npm:^3.8.3":
|
||||
version: 3.8.3
|
||||
resolution: "superagent@npm:3.8.3"
|
||||
@@ -46345,6 +46438,16 @@ __metadata:
|
||||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
"supertest@npm:^7.1.3":
|
||||
version: 7.1.3
|
||||
resolution: "supertest@npm:7.1.3"
|
||||
dependencies:
|
||||
methods: "npm:^1.1.2"
|
||||
superagent: "npm:^10.2.2"
|
||||
checksum: 10/d148d05ed52e2cd487c483aae8721bf83f2611eb630d847b1cb37b3f1b09c499aa6b6446d3739c1db2459df1668bf089870437f71e57fc39c7194f51bdac6119
|
||||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
"supports-color@npm:8.1.1, supports-color@npm:^8.0.0, supports-color@npm:^8.1.0, supports-color@npm:^8.1.1, supports-color@npm:~8.1.1":
|
||||
version: 8.1.1
|
||||
resolution: "supports-color@npm:8.1.1"
|
||||
|
||||
Reference in New Issue
Block a user