diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index d1e50b61a..c997481bc 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -198,7 +198,20 @@ jobs: test-preview-service: name: Preview service runs-on: blacksmith - if: false # disabled as there is nothing to run + container: + image: ghcr.io/specklesystems/speckle-ubuntu-chromium:latest + services: + redis: + image: redis:7.2.4 + options: >- + --health-cmd "redis-cli ping" + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + - 6379:6379 + env: + REDIS_URL: redis://redis:6379 steps: - uses: actions/checkout@v4.2.2 - uses: useblacksmith/setup-node@v5 @@ -206,9 +219,19 @@ jobs: node-version: 22 cache: yarn - name: Install dependencies - run: YARN_ENABLE_HARDENED_MODE=0 PUPPETEER_SKIP_DOWNLOAD=true PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD=1 yarn --immutable + run: YARN_ENABLE_HARDENED_MODE=0 PUPPETEER_SKIP_DOWNLOAD=true PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD=0 yarn --immutable - name: Build public packages run: yarn build:public + - run: dbus-daemon --system &> /dev/null + - run: cp .env.test-example .env.test + working-directory: 'packages/preview-service' + - run: yarn build:frontend + working-directory: 'packages/preview-service' + - run: yarn link:frontend + working-directory: 'packages/preview-service' + - name: Run tests + run: yarn test:ci + working-directory: 'packages/preview-service' docker-build-postgres-container: runs-on: blacksmith diff --git a/packages/preview-service/.env.example b/packages/preview-service/.env.example index 503cd0a6e..9787b9425 100644 --- a/packages/preview-service/.env.example +++ b/packages/preview-service/.env.example @@ -11,4 +11,4 @@ LOG_PRETTY='true' # Local dev settings ########################################################## # Uncomment to enable pino-pretty log formatting in debug mode (disabled cause of node22 issues) -# ALLOW_PRETTY_DEBUGGER=true \ No newline at end of file +# ALLOW_PRETTY_DEBUGGER=true diff --git a/packages/preview-service/.env.test-example b/packages/preview-service/.env.test-example new file mode 100644 index 000000000..1c0e8497d --- /dev/null +++ b/packages/preview-service/.env.test-example @@ -0,0 +1,11 @@ +PREVIEWS_HEADED='true' +CHROMIUM_EXECUTABLE_PATH='/usr/bin/chromium' +USER_DATA_DIR='/tmp/puppeteer' +REDIS_URL='redis://localhost:6379' +PORT='3099' +PROMETHEUS_METRICS_PORT='' +PREVIEWS_HEADED='false' + +########################################################## +# Test settings +########################################################## diff --git a/packages/preview-service/.gitignore b/packages/preview-service/.gitignore index d70ebaa1d..c43878203 100644 --- a/packages/preview-service/.gitignore +++ b/packages/preview-service/.gitignore @@ -1 +1,3 @@ -public \ No newline at end of file +public +tests/snapshots/diff.png +tests/snapshots/result.png diff --git a/packages/preview-service/package.json b/packages/preview-service/package.json index 9e41d4b06..f384ed61f 100644 --- a/packages/preview-service/package.json +++ b/packages/preview-service/package.json @@ -19,7 +19,8 @@ "link:frontend": "yarn build:frontend && rimraf ./public && ln -s ../preview-frontend/dist ./public", "dev": "tsx --env-file=.env --watch src/main.ts", "publishTask": "tsx --env-file=.env scripts/publishTask.ts", - "test": "echo 'no tests configured'", + "test": "NODE_ENV=test vitest run --sequence.shuffle", + "test:ci": "NODE_ENV=test LOG_LEVEL=silent LOG_PRETTY=true vitest run --sequence.shuffle", "lint": "yarn lint:tsc && yarn lint:eslint", "lint:ci": "yarn lint:tsc", "lint:tsc": "tsc --noEmit", @@ -46,14 +47,19 @@ "@swc/core": "^1.9.3", "@types/express": "^4.17.13", "@types/node": "^18.19.38", + "@types/pngjs": "^6.0.5", "eslint": "^9.4.0", "eslint-config-prettier": "^9.1.0", "eslint-plugin-vitest": "^0.5.4", + "pixelmatch": "^7.1.0", + "pngjs": "^7.0.0", "prettier": "^2.5.1", "rimraf": "^6.0.1", + "supertest": "^7.1.3", "ts-node": "^10.9.2", "tsx": "^4.19.2", "typescript": "^4.6.4", - "typescript-eslint": "^7.12.0" + "typescript-eslint": "^7.12.0", + "vitest": "^1.6.0" } } diff --git a/packages/preview-service/src/bootstrap.ts b/packages/preview-service/src/bootstrap.ts index deda5b6ba..60db1445f 100644 --- a/packages/preview-service/src/bootstrap.ts +++ b/packages/preview-service/src/bootstrap.ts @@ -2,8 +2,6 @@ import generateAliasesResolver from 'esm-module-alias' import path from 'node:path' import fs from 'node:fs' import { fileURLToPath } from 'url' -import dotenv from 'dotenv' -dotenv.config() /** * Singleton module for src root and package root directory resolution diff --git a/packages/preview-service/src/config.ts b/packages/preview-service/src/config.ts index 5f2b04e4c..ab9cdf92f 100644 --- a/packages/preview-service/src/config.ts +++ b/packages/preview-service/src/config.ts @@ -1,3 +1,9 @@ +import dotenv from 'dotenv' + +export const isTestEnvironment = () => process.env['NODE_ENV'] === 'test' + +dotenv.config(isTestEnvironment() ? { path: '.env.test' } : {}) + import { z } from 'zod' import { parseEnv } from 'znv' diff --git a/packages/preview-service/src/main.ts b/packages/preview-service/src/main.ts index c8490497e..0f2bee50e 100644 --- a/packages/preview-service/src/main.ts +++ b/packages/preview-service/src/main.ts @@ -1,232 +1,15 @@ import express from 'express' -import puppeteer, { Browser } from 'puppeteer' -import { createTerminus } from '@godaddy/terminus' -import type { Logger } from 'pino' -import type Bull from 'bull' - -import { JobPayload, PreviewResultPayload } from '@speckle/shared/workers/previews' -import { AppState } from '@speckle/shared/workers' -import { - REDIS_URL, - HOST, - PORT, - CHROMIUM_EXECUTABLE_PATH, - PREVIEWS_HEADED, - USER_DATA_DIR, - PREVIEW_TIMEOUT, - GPU_ENABLED -} from '@/config.js' -import { logger } from '@/logging.js' -import { jobProcessor } from '@/jobProcessor.js' +import { isTestEnvironment } from '@/config.js' import { initMetrics, initPrometheusRegistry } from '@/metrics.js' -import { ensureError } from '@speckle/shared' -import { initializeQueue } from '@speckle/shared/queue' -import { isRedisReady } from '@speckle/shared/redis' +import { buildServer, initServer } from './server' const app = express() -const host = HOST -const port = PORT - -const JobQueueName = 'preview-service-jobs' - -let appState: AppState = AppState.STARTING // serve the preview-frontend app.use(express.static('public')) -await initMetrics({ app, registry: initPrometheusRegistry() }) -let jobQueue: Bull.Queue | undefined = undefined -// store this callback, so on shutdown we can error the job -let currentJob: { logger: Logger; done: Bull.DoneCallback } | undefined = undefined +if (!isTestEnvironment()) await initMetrics({ app, registry: initPrometheusRegistry() }) -// browser is a global variable, so we can handle the shutdown of the browser -// in the beforeShutdown function. We need to stop processing jobs before we -// can close the browser -let browser: Browser | undefined = undefined +const server = buildServer({ app }) -const server = app.listen(port, host, async () => { - logger.info({ port }, '📡 Started Preview Service server, listening on {port}') - appState = AppState.RUNNING - - const gpuArgs = ['--use-gl=angle', '--use-angle=gl-egl'] - - const launchBrowser = async (): Promise => { - const launchArguments = [ - '--no-sandbox', - '--disable-setuid-sandbox', - '--disable-dev-shm-usage', - '--disable-session-crashed-bubble', - ...(GPU_ENABLED ? gpuArgs : []) - ] - logger.debug( - `Starting browser, located at "${CHROMIUM_EXECUTABLE_PATH}", with the following arguments: ${JSON.stringify( - launchArguments - )}` - ) - return await puppeteer.launch({ - headless: !PREVIEWS_HEADED, - executablePath: CHROMIUM_EXECUTABLE_PATH, - userDataDir: USER_DATA_DIR, - // slowMo: 3000, // Use for debugging during development - // we trust the web content that is running, so can disable the sandbox - // disabling the sandbox allows us to run the docker image without linux kernel privileges - args: launchArguments, - protocolTimeout: PREVIEW_TIMEOUT, - // handle closing of the browser by the preview-service, not puppeteer - // this is important for the preview-service to be able to shut down gracefully, - // otherwise we end up in race condition where puppeteer closes before preview-service - handleSIGHUP: false, - handleSIGINT: false, - handleSIGTERM: false - }) - } - - try { - jobQueue = await initializeQueue({ - queueName: JobQueueName, - redisUrl: REDIS_URL - }) - } catch (e) { - const err = ensureError(e, 'Unknown error creating job queue') - logger.error({ err }, 'Error creating job queue') - - // the callback to server.listen has failed, so we need to exit the process and not just return - await beforeShutdown() // handle the shutdown gracefully - await onShutdown() - process.exit(1) - } - - logger.debug(`Starting processing of "${JobQueueName}" message queue`) - - // nothing after this line is getting called, this blocks - await jobQueue.process(async (payload, done) => { - let encounteredError = false - let jobLogger = logger.child({ - payloadId: payload.id, - jobPriorAttemptsMade: payload.attemptsMade - }) - - if (browser) { - const message = 'Tried to start job but Browser is already open.' - done(new Error(message)) - throw new Error(message) - } - - try { - currentJob = { done, logger: jobLogger } - const job = payload.data - jobLogger = jobLogger.child({ - jobId: job.jobId, - serverUrl: job.url - }) - const resultsQueue = await initializeQueue({ - queueName: job.responseQueue, - redisUrl: REDIS_URL - }) - - browser = await launchBrowser() - const result = await jobProcessor({ - logger: jobLogger, - browser, - job, - port: PORT, - timeout: PREVIEW_TIMEOUT, - getAppState: () => appState - }) - - // with removeOnComplete, the job response potentially containing a large images, - // is cleared from the response queue - await resultsQueue.add(result, { removeOnComplete: true }) - } catch (err) { - if (appState === AppState.SHUTTINGDOWN) { - // likely that the job was cancelled due to the service shutting down - jobLogger.warn({ err }, 'Processing job {jobId} failed') - } else { - jobLogger.error({ err }, 'Processing job {jobId} failed') - } - if (err instanceof Error) { - encounteredError = true - done(err) - } else { - throw err - } - } finally { - if (browser) await browser.close() - browser = undefined - if (!encounteredError) done() - currentJob = undefined - } - }) -}) - -const beforeShutdown = async () => { - logger.info('🛑 Beginning shut down, pausing all jobs') - appState = AppState.SHUTTINGDOWN - // stop accepting new jobs and kill any running jobs - if (jobQueue) { - await jobQueue.pause( - true, // just pausing this local worker of the queue - true // do not wait for active jobs to finish - ) - } - - if (currentJob) { - currentJob.logger.warn('Cancelling job due to preview-service shutdown') - currentJob.done(new Error('Job cancelled due to preview-service shutdown')) - } - if (browser) { - // preview-service is responsible for closing the browser - // to allow us to stop listening for new jobs and properly respond to any - // current job before we kill the browser - logger.info('Closing browser') - await browser.close() - browser = undefined - } - // no need to close the job queue and redis client, when the process exits they will be closed automatically -} - -const onShutdown = async () => { - logger.info('👋 Completed shut down, now exiting') -} - -createTerminus(server, { - healthChecks: { - '/liveness': () => Promise.resolve('ok'), - '/readiness': async (args: { state: { isShuttingDown: boolean } }) => { - const { isShuttingDown } = args.state - if (isShuttingDown) { - return Promise.reject(new Error('Preview service is shutting down')) - } - - if (!jobQueue) { - return Promise.reject(new Error('Job queue is not initialized')) - } - - try { - await isRedisReady(jobQueue.client) - } catch (e) { - return Promise.reject( - ensureError(e, 'Unknown error when checking Redis client') - ) - } - const isReady = await jobQueue.isReady() - if (!isReady) - return Promise.reject( - new Error( - 'Preview service is not ready. Redis or Bull is not either reachable or ready.' - ) - ) - - return Promise.resolve('ok') - } - }, - beforeShutdown, - onShutdown, - logger: (msg, err) => { - if (err) { - logger.error({ err }, msg) - return - } - logger.info(msg) - } -}) +initServer(server) diff --git a/packages/preview-service/src/server.ts b/packages/preview-service/src/server.ts new file mode 100644 index 000000000..c1470dcd4 --- /dev/null +++ b/packages/preview-service/src/server.ts @@ -0,0 +1,229 @@ +import puppeteer, { Browser } from 'puppeteer' +import { JobPayload, PreviewResultPayload } from '@speckle/shared/workers/previews' +import { AppState } from '@speckle/shared/workers' +import type Bull from 'bull' +import { + REDIS_URL, + HOST, + PORT, + CHROMIUM_EXECUTABLE_PATH, + PREVIEWS_HEADED, + USER_DATA_DIR, + PREVIEW_TIMEOUT, + GPU_ENABLED +} from '@/config.js' + +import { jobProcessor } from '@/jobProcessor.js' +import { initializeQueue } from '@speckle/shared/queue' +import { Express } from 'express' +import { logger } from '@/logging.js' +import { Logger } from 'pino' +const JobQueueName = 'preview-service-jobs' +import { ensureError } from '@speckle/shared' +import { createTerminus } from '@godaddy/terminus' +import { isRedisReady } from '@speckle/shared/redis' +import { Server } from 'http' + +let appState: AppState = AppState.STARTING + +let jobQueue: Bull.Queue | undefined = undefined + +// store this callback, so on shutdown we can error the job +let currentJob: { logger: Logger; done: Bull.DoneCallback } | undefined = undefined + +// browser is a global variable, so we can handle the shutdown of the browser +// in the beforeShutdown function. We need to stop processing jobs before we +// can close the browser +let browser: Browser | undefined = undefined + +export const buildServer = ({ + port, + host, + app +}: Partial<{ port: number; host: string }> & { app: Express }) => + app.listen(port || PORT, host || HOST, async () => { + logger.info({ port }, '📡 Started Preview Service server, listening on {port}') + appState = AppState.RUNNING + + const gpuArgs = ['--use-gl=angle', '--use-angle=gl-egl'] + + const launchBrowser = async (): Promise => { + const launchArguments = [ + '--no-sandbox', + '--disable-setuid-sandbox', + '--disable-dev-shm-usage', + '--disable-session-crashed-bubble', + ...(GPU_ENABLED ? gpuArgs : []) + ] + logger.debug( + `Starting browser, located at "${CHROMIUM_EXECUTABLE_PATH}", with the following arguments: ${JSON.stringify( + launchArguments + )}` + ) + return await puppeteer.launch({ + headless: !PREVIEWS_HEADED, + executablePath: CHROMIUM_EXECUTABLE_PATH, + userDataDir: USER_DATA_DIR, + // slowMo: 3000, // Use for debugging during development + // we trust the web content that is running, so can disable the sandbox + // disabling the sandbox allows us to run the docker image without linux kernel privileges + args: launchArguments, + protocolTimeout: PREVIEW_TIMEOUT, + // handle closing of the browser by the preview-service, not puppeteer + // this is important for the preview-service to be able to shut down gracefully, + // otherwise we end up in race condition where puppeteer closes before preview-service + handleSIGHUP: false, + handleSIGINT: false, + handleSIGTERM: false + }) + } + + try { + jobQueue = await initializeQueue({ + queueName: JobQueueName, + redisUrl: REDIS_URL + }) + } catch (e) { + const err = ensureError(e, 'Unknown error creating job queue') + logger.error({ err }, 'Error creating job queue') + + // the callback to server.listen has failed, so we need to exit the process and not just return + await beforeShutdown() // handle the shutdown gracefully + await onShutdown() + process.exit(1) + } + logger.debug(`Starting processing of "${JobQueueName}" message queue`) + + // nothing after this line is getting called, this blocks + await jobQueue.process(async (payload, done) => { + let encounteredError = false + let jobLogger = logger.child({ + payloadId: payload.id, + jobPriorAttemptsMade: payload.attemptsMade + }) + + if (browser) { + const message = 'Tried to start job but Browser is already open.' + done(new Error(message)) + throw new Error(message) + } + + try { + currentJob = { done, logger: jobLogger } + const job = payload.data + jobLogger = jobLogger.child({ + jobId: job.jobId, + serverUrl: job.url + }) + const resultsQueue = await initializeQueue({ + queueName: job.responseQueue, + redisUrl: REDIS_URL + }) + + browser = await launchBrowser() + const result = await jobProcessor({ + logger: jobLogger, + browser, + job, + port: PORT, + timeout: PREVIEW_TIMEOUT, + getAppState: () => appState + }) + + // with removeOnComplete, the job response potentially containing a large images, + // is cleared from the response queue + await resultsQueue.add(result, { removeOnComplete: true }) + } catch (err) { + if (appState === AppState.SHUTTINGDOWN) { + // likely that the job was cancelled due to the service shutting down + jobLogger.warn({ err }, 'Processing job {jobId} failed') + } else { + jobLogger.error({ err }, 'Processing job {jobId} failed') + } + if (err instanceof Error) { + encounteredError = true + done(err) + } else { + throw err + } + } finally { + if (browser) await browser.close() + browser = undefined + if (!encounteredError) done() + currentJob = undefined + } + }) + }) + +export const initServer = (server: Server) => + createTerminus(server, { + healthChecks: { + '/liveness': () => Promise.resolve('ok'), + '/readiness': isReady + }, + beforeShutdown, + onShutdown, + logger: (msg, err) => { + if (err) { + logger.error({ err }, msg) + return + } + logger.info(msg) + } + }) + +const beforeShutdown = async () => { + logger.info('🛑 Beginning shut down, pausing all jobs') + appState = AppState.SHUTTINGDOWN + // stop accepting new jobs and kill any running jobs + if (jobQueue) { + await jobQueue.pause( + true, // just pausing this local worker of the queue + true // do not wait for active jobs to finish + ) + } + + if (currentJob) { + currentJob.logger.warn('Cancelling job due to preview-service shutdown') + currentJob.done(new Error('Job cancelled due to preview-service shutdown')) + } + if (browser) { + // preview-service is responsible for closing the browser + // to allow us to stop listening for new jobs and properly respond to any + // current job before we kill the browser + logger.info('Closing browser') + await browser.close() + browser = undefined + } + // no need to close the job queue and redis client, when the process exits they will be closed automatically +} + +const onShutdown = async () => { + logger.info('👋 Completed shut down, now exiting') +} + +const isReady = async (args: { state: { isShuttingDown: boolean } }) => { + const { isShuttingDown } = args.state + if (isShuttingDown) { + return Promise.reject(new Error('Preview service is shutting down')) + } + + if (!jobQueue) { + return Promise.reject(new Error('Job queue is not initialized')) + } + + try { + await isRedisReady(jobQueue.client) + } catch (e) { + return Promise.reject(ensureError(e, 'Unknown error when checking Redis client')) + } + const isReady = await jobQueue.isReady() + if (!isReady) + return Promise.reject( + new Error( + 'Preview service is not ready. Redis or Bull is not either reachable or ready.' + ) + ) + + return Promise.resolve('ok') +} diff --git a/packages/preview-service/tests/main.spec.ts b/packages/preview-service/tests/main.spec.ts new file mode 100644 index 000000000..7187c9544 --- /dev/null +++ b/packages/preview-service/tests/main.spec.ts @@ -0,0 +1,141 @@ +import { describe, it, beforeAll, afterAll, expect } from 'vitest' +import express from 'express' +import { buildServer } from 'src/server' +import { Server } from 'http' +import { initializeQueue } from '@speckle/shared/queue' +import { REDIS_URL } from '@/config.js' +import Bull from 'bull' +import { JobPayload } from '@speckle/shared/workers/previews' +import { randomUUID } from 'crypto' +import supertest, { SuperTest, Test } from 'supertest' +import { TIME_MS } from '@speckle/shared' +import fs from 'fs' +import pixelmatch from 'pixelmatch' +import { PNG } from 'pngjs' +import path from 'path' + +const BASE_IMAGE = path.resolve(__dirname, 'snapshots/base.png') +const TEST_RESULT = path.resolve(__dirname, 'snapshots/result.png') +const DIFF = path.resolve(__dirname, 'snapshots/diff.png') + +describe('preview-service', () => { + let server: Server + let request: SuperTest + let jobQueue: Bull.Queue + let responseQueue: Bull.Queue<{ + jobId: string + status: string + result: { + screenshots: object + } + }> + + const testId = randomUUID() + const JOB_QUEUE = 'preview-service-jobs' + const RESPONSE_QUEUE = 'preview-service-jobs-test-queue-' + testId + + const sleep = async (ms: number) => { + await new Promise((resolve) => setTimeout(resolve, ms)) + } + + beforeAll(async () => { + const app = express() + app.use(express.static('public')) + request = supertest(app) + server = buildServer({ app }) + + jobQueue = await initializeQueue({ + queueName: JOB_QUEUE, + redisUrl: REDIS_URL + }) + + responseQueue = await initializeQueue({ + queueName: RESPONSE_QUEUE, + redisUrl: REDIS_URL + }) + + // delete existing images + fs.rmSync(TEST_RESULT, { recursive: false, force: true }) + fs.rmSync(DIFF, { recursive: false, force: true }) + + // TODO: remove this head start + // only awaiting for the server to start does not work + // we should await the start of the job processing + await sleep(6 * TIME_MS.second) + }) + + afterAll(async () => { + server.close() + await jobQueue.close() + await responseQueue.close() + }) + + it('inits a server', async () => { + expect(server).to.be.instanceOf(Server) + }) + + it('hits the server', async () => { + const response = await request.get('/') + + expect(response.status).to.equal(200) + }) + + it('process a rendering task providing back the image', async () => { + const ID = 'test-job' + testId + + await jobQueue.add({ + url: 'https://latest.speckle.systems/projects/8b94a55ee5/models/7f98c5b62e', + token: '', + jobId: ID, + responseQueue: RESPONSE_QUEUE + }) + + let jobs = null + while (!jobs || !jobs.length) { + // can run until the test suite times out + jobs = await responseQueue.getJobs(['waiting']) + + await sleep(1 * TIME_MS.second) + } + + expect(jobs).to.have.lengthOf(1) + const [job] = jobs + expect(job.data.jobId).to.equal(ID) + expect(job.data.status).to.equal('success') + expect(job.data.result).to.be.an('object') + expect(job.data.result.screenshots).toBeDefined() + + // write the image to a result file, for debugging + + const image = + '0' in job.data.result.screenshots + ? (job.data.result.screenshots['0'] as string) + : null + + if (!image) expect.fail('No image found') + + const clean = Buffer.from(image.replace(/^data:image\/png;base64,/, ''), 'base64') + + fs.writeFileSync(TEST_RESULT, clean) + + // test max diff + + const base = PNG.sync.read(fs.readFileSync(BASE_IMAGE)) + const result = PNG.sync.read(fs.readFileSync(TEST_RESULT)) + const diff = new PNG({ width: base.width, height: base.height }) + const totalPixels = base.width * base.height + + const diffPixels = pixelmatch( + base.data, + result.data, + diff.data, + base.width, + base.height, + { threshold: 0.1 } + ) + + fs.writeFileSync(DIFF, PNG.sync.write(diff)) + const diffPercentage = Number(((diffPixels / totalPixels) * 100).toFixed(2)) + expect(diffPercentage).to.be.lessThan(10) + }) +}) diff --git a/packages/preview-service/tests/snapshots/base.png b/packages/preview-service/tests/snapshots/base.png new file mode 100644 index 000000000..0b2ce9d01 Binary files /dev/null and b/packages/preview-service/tests/snapshots/base.png differ diff --git a/packages/preview-service/vitest.config.ts b/packages/preview-service/vitest.config.ts new file mode 100644 index 000000000..bf52f2daf --- /dev/null +++ b/packages/preview-service/vitest.config.ts @@ -0,0 +1,20 @@ +import { TIME_MS } from '@speckle/shared' +import path from 'path' +import { configDefaults, defineConfig } from 'vitest/config' + +export default defineConfig({ + test: { + exclude: [...configDefaults.exclude], + // reporters: ['verbose', 'hanging-process'] //uncomment to debug hanging processes etc. + sequence: { + shuffle: true, + concurrent: true + }, + testTimeout: 2 * TIME_MS.minute + }, + resolve: { + alias: { + '@': path.resolve(__dirname, './src') + } + } +}) diff --git a/utils/ubuntu-chromium/Dockerfile b/utils/ubuntu-chromium/Dockerfile new file mode 100644 index 000000000..b9a93d4f6 --- /dev/null +++ b/utils/ubuntu-chromium/Dockerfile @@ -0,0 +1,30 @@ +FROM node:20-slim + +# hadolint ignore=DL3008 +RUN apt-get update && \ + apt-get install -y \ + chromium \ + fonts-liberation \ + libappindicator3-1 \ + libasound2 \ + libatk-bridge2.0-0 \ + libatk1.0-0 \ + libcups2 \ + libdbus-1-3 \ + libgdk-pixbuf2.0-0 \ + libnspr4 \ + libnss3 \ + libxcomposite1 \ + libxdamage1 \ + libxrandr2 \ + xdg-utils \ + wget \ + ca-certificates \ + --no-install-recommends && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +ENV PUPPETEER_EXECUTABLE_PATH=/usr/bin/chromium + +# Create app directory +WORKDIR /usr/src/app diff --git a/yarn.lock b/yarn.lock index 31c358218..d76ec45ce 100644 --- a/yarn.lock +++ b/yarn.lock @@ -12731,6 +12731,13 @@ __metadata: languageName: node linkType: hard +"@noble/hashes@npm:^1.1.5": + version: 1.8.0 + resolution: "@noble/hashes@npm:1.8.0" + checksum: 10/474b7f56bc6fb2d5b3a42132561e221b0ea4f91e590f4655312ca13667840896b34195e2b53b7f097ec080a1fdd3b58d902c2a8d0fbdf51d2e238b53808a177e + languageName: node + linkType: hard + "@nodelib/fs.scandir@npm:2.1.5": version: 2.1.5 resolution: "@nodelib/fs.scandir@npm:2.1.5" @@ -13586,6 +13593,15 @@ __metadata: languageName: node linkType: hard +"@paralleldrive/cuid2@npm:^2.2.2": + version: 2.2.2 + resolution: "@paralleldrive/cuid2@npm:2.2.2" + dependencies: + "@noble/hashes": "npm:^1.1.5" + checksum: 10/40ee269d6e47b4fed7706a2e4da7c27c3c668ebc969110d6d112277b6b16a67cce0503b53b9943f2c55035a72d225f77ea5541e03396d6429eec9252137a53b7 + languageName: node + linkType: hard + "@parcel/watcher-android-arm64@npm:2.4.1": version: 2.4.1 resolution: "@parcel/watcher-android-arm64@npm:2.4.1" @@ -16191,6 +16207,7 @@ __metadata: "@swc/core": "npm:^1.9.3" "@types/express": "npm:^4.17.13" "@types/node": "npm:^18.19.38" + "@types/pngjs": "npm:^6.0.5" bull: "npm:^4.16.4" dotenv: "npm:^16.4.7" eslint: "npm:^9.4.0" @@ -16201,14 +16218,18 @@ __metadata: pino: "npm:^8.7.0" pino-http: "npm:^8.6.1" pino-pretty: "npm:^9.1.1" + pixelmatch: "npm:^7.1.0" + pngjs: "npm:^7.0.0" prettier: "npm:^2.5.1" prom-client: "npm:^14.0.1" puppeteer: "npm:^23.9.0" rimraf: "npm:^6.0.1" + supertest: "npm:^7.1.3" ts-node: "npm:^10.9.2" tsx: "npm:^4.19.2" typescript: "npm:^4.6.4" typescript-eslint: "npm:^7.12.0" + vitest: "npm:^1.6.0" znv: "npm:^0.4.0" zod: "npm:^3.23.8" languageName: unknown @@ -19594,6 +19615,15 @@ __metadata: languageName: node linkType: hard +"@types/pngjs@npm:^6.0.5": + version: 6.0.5 + resolution: "@types/pngjs@npm:6.0.5" + dependencies: + "@types/node": "npm:*" + checksum: 10/132fce25817d47a784ed48aa678332521b0f7e6edbaa76f3fa4e9ca1228078788ae712f99ad4d1a324d9ba0b14829958677eabf3ebef1fb6e120816f433f0cd8 + languageName: node + linkType: hard + "@types/polylabel@npm:1.1.3": version: 1.1.3 resolution: "@types/polylabel@npm:1.1.3" @@ -22255,7 +22285,7 @@ __metadata: languageName: node linkType: hard -"asap@npm:~2.0.3": +"asap@npm:^2.0.0, asap@npm:~2.0.3": version: 2.0.6 resolution: "asap@npm:2.0.6" checksum: 10/b244c0458c571945e4b3be0b14eb001bea5596f9868cc50cc711dc03d58a7e953517d3f0dad81ccde3ff37d1f074701fa76a6f07d41aaa992d7204a37b915dda @@ -24874,6 +24904,13 @@ __metadata: languageName: node linkType: hard +"component-emitter@npm:^1.3.0": + version: 1.3.1 + resolution: "component-emitter@npm:1.3.1" + checksum: 10/94550aa462c7bd5a61c1bc480e28554aa306066930152d1b1844a0dd3845d4e5db7e261ddec62ae184913b3e59b55a2ad84093b9d3596a8f17c341514d6c483d + languageName: node + linkType: hard + "compress-commons@npm:^6.0.2": version: 6.0.2 resolution: "compress-commons@npm:6.0.2" @@ -25223,7 +25260,7 @@ __metadata: languageName: node linkType: hard -"cookiejar@npm:^2.1.0, cookiejar@npm:^2.1.1": +"cookiejar@npm:^2.1.0, cookiejar@npm:^2.1.1, cookiejar@npm:^2.1.4": version: 2.1.4 resolution: "cookiejar@npm:2.1.4" checksum: 10/4a184f5a0591df8b07d22a43ea5d020eacb4572c383e853a33361a99710437eaa0971716c688684075bbf695b484f5872e9e3f562382e46858716cb7fc8ce3f4 @@ -26725,6 +26762,16 @@ __metadata: languageName: node linkType: hard +"dezalgo@npm:^1.0.4": + version: 1.0.4 + resolution: "dezalgo@npm:1.0.4" + dependencies: + asap: "npm:^2.0.0" + wrappy: "npm:1" + checksum: 10/895389c6aead740d2ab5da4d3466d20fa30f738010a4d3f4dcccc9fc645ca31c9d10b7e1804ae489b1eb02c7986f9f1f34ba132d409b043082a86d9a4e745624 + languageName: node + linkType: hard + "didyoumean@npm:^1.2.2": version: 1.2.2 resolution: "didyoumean@npm:1.2.2" @@ -30197,6 +30244,17 @@ __metadata: languageName: node linkType: hard +"formidable@npm:^3.5.4": + version: 3.5.4 + resolution: "formidable@npm:3.5.4" + dependencies: + "@paralleldrive/cuid2": "npm:^2.2.2" + dezalgo: "npm:^1.0.4" + once: "npm:^1.4.0" + checksum: 10/4645e6ce3d8bbefd3dd873dcd6211362da3bf8a04c8426d7f454c238be0142975f02e5bdbc792fdbd2be493fdcf5442fe01d9a246bd8c3fd8e779738290cc630 + languageName: node + linkType: hard + "forwarded@npm:0.2.0": version: 0.2.0 resolution: "forwarded@npm:0.2.0" @@ -36614,7 +36672,7 @@ __metadata: languageName: node linkType: hard -"mime@npm:^2.4.6": +"mime@npm:2.6.0, mime@npm:^2.4.6": version: 2.6.0 resolution: "mime@npm:2.6.0" bin: @@ -40669,6 +40727,17 @@ __metadata: languageName: node linkType: hard +"pixelmatch@npm:^7.1.0": + version: 7.1.0 + resolution: "pixelmatch@npm:7.1.0" + dependencies: + pngjs: "npm:^7.0.0" + bin: + pixelmatch: bin/pixelmatch + checksum: 10/57a122196318ea8ce74e8759b1b7b94b9f9627b495cd79e50a49d470dc23b6c679e89c38660d0f7e8f959eac3b279c55b728e52d02c276dc51505f06eaba1141 + languageName: node + linkType: hard + "pkg-dir@npm:^3.0.0": version: 3.0.0 resolution: "pkg-dir@npm:3.0.0" @@ -40807,6 +40876,13 @@ __metadata: languageName: node linkType: hard +"pngjs@npm:^7.0.0": + version: 7.0.0 + resolution: "pngjs@npm:7.0.0" + checksum: 10/e843ebbb0df092ee0f3a3e7dbd91ff87a239a4e4c4198fff202916bfb33b67622f4b83b3c29f3ccae94fcb97180c289df06068624554f61686fe6b9a4811f7db + languageName: node + linkType: hard + "polished@npm:^4.2.2": version: 4.2.2 resolution: "polished@npm:4.2.2" @@ -46308,6 +46384,23 @@ __metadata: languageName: node linkType: hard +"superagent@npm:^10.2.2": + version: 10.2.2 + resolution: "superagent@npm:10.2.2" + dependencies: + component-emitter: "npm:^1.3.0" + cookiejar: "npm:^2.1.4" + debug: "npm:^4.3.4" + fast-safe-stringify: "npm:^2.1.1" + form-data: "npm:^4.0.0" + formidable: "npm:^3.5.4" + methods: "npm:^1.1.2" + mime: "npm:2.6.0" + qs: "npm:^6.11.0" + checksum: 10/e89ae49163df0db50e6a77316a7304a16640df11a8d2219bef11e69f59c74e54c16670ec250c2ab59f06887f71bb5d6e5933f735b931cbdbe34ba490d78b5d70 + languageName: node + linkType: hard + "superagent@npm:^3.7.0, superagent@npm:^3.8.3": version: 3.8.3 resolution: "superagent@npm:3.8.3" @@ -46345,6 +46438,16 @@ __metadata: languageName: node linkType: hard +"supertest@npm:^7.1.3": + version: 7.1.3 + resolution: "supertest@npm:7.1.3" + dependencies: + methods: "npm:^1.1.2" + superagent: "npm:^10.2.2" + checksum: 10/d148d05ed52e2cd487c483aae8721bf83f2611eb630d847b1cb37b3f1b09c499aa6b6446d3739c1db2459df1668bf089870437f71e57fc39c7194f51bdac6119 + languageName: node + linkType: hard + "supports-color@npm:8.1.1, supports-color@npm:^8.0.0, supports-color@npm:^8.1.0, supports-color@npm:^8.1.1, supports-color@npm:~8.1.1": version: 8.1.1 resolution: "supports-color@npm:8.1.1"