diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index a2e4c1b15..5f65b7580 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -2,21 +2,48 @@ // README at: https://github.com/devcontainers/templates/tree/main/src/typescript-node { "name": "Node.js & TypeScript", - // Or use a Dockerfile or Docker Compose file. More info: https://containers.dev/guide/dockerfile - "image": "mcr.microsoft.com/devcontainers/typescript-node:1-18-bullseye", + + // We want to deploy all the dependencies (Postgres, Redis, etc.) alongside the devcontainer + // so we can run the app in the container. + "dockerComposeFile": [ + "../docker-compose-deps.yml", + "./docker-compose-devcontainer.yml" + ], + // name of the service within the docker-compose file which devcontainer tools should connect to + "service": "devcontainer", + //services in the docker-compose file which should run when the devcontainer starts + //"runServices": [] // defaults to all + + // Path to the workspace within the container. + // Needs to match destination volume ('volumes' property) in docker-compose-devcontainer.yml. + "workspaceFolder": "/workspaces/${localWorkspaceFolderBasename}", // Features to add to the dev container. More info: https://containers.dev/features. - // "features": {}, + "features": { + "ghcr.io/devcontainers/features/docker-outside-of-docker:1": {} + }, - // Use 'forwardPorts' to make a list of ports inside the container available locally. - // "forwardPorts": [], + // Use 'forwardPorts' to make a list of ports inside the container available locally (e.g. via your browser on your desktop) + "forwardPorts": [ + 3000, //speckle server + 6006, //storybook + 8081 //speckle frontend + ], // Use 'postCreateCommand' to run commands after the container is created. - "postCreateCommand": "yarn && yarn build:public" + "postCreateCommand": "yarn && yarn build:public && cp -n /workspaces/${localWorkspaceFolderBasename}/packages/server/.env-example /workspaces/${localWorkspaceFolderBasename}/packages/server/.env && cp -n /workspaces/${localWorkspaceFolderBasename}/packages/frontend-2/.env.example /workspaces/${localWorkspaceFolderBasename}/packages/frontend-2/.env", // Configure tool-specific properties. // "customizations": {}, // Uncomment to connect as root instead. More info: https://aka.ms/dev-containers-non-root. // "remoteUser": "root" + + "hostRequirements": { + "cpus": 4, + "memory": "8gb" + }, + + // Environment variables to set in the container. + "remoteEnv": { "LOCAL_WORKSPACE_FOLDER": "${localWorkspaceFolder}" } } diff --git a/.devcontainer/docker-compose-devcontainer.yml b/.devcontainer/docker-compose-devcontainer.yml new file mode 100644 index 000000000..6562ba975 --- /dev/null +++ b/.devcontainer/docker-compose-devcontainer.yml @@ -0,0 +1,194 @@ +services: + devcontainer: + image: 'mcr.microsoft.com/devcontainers/typescript-node:1-18-bullseye' + volumes: + - ..:/workspaces:cached # mounts our local directory (git root) to the container in /workspaces directory (with access mode of cached) + + # We need to override all node_modules directories + # by mounting an empty volume to avoid conflicting + # CPU build instructions between the host machine and the DevContainer's docker image + # (this allows the devcontainer to be based on linux yet work on Apple Silicon etc..) + # If you add a new package with a new `node_modules`, it needs to be added here + - type: volume + source: node_modules # top-level volume + target: /workspaces/speckle-server/node_modules + read_only: false + - type: volume + source: dui3-node_modules + target: /workspaces/speckle-server/packages/dui3/node_modules + read_only: false + - type: volume + source: fileimport-service-node_modules + target: /workspaces/speckle-server/packages/fileimport-service/node_modules + read_only: false + - type: volume + source: frontend-2-node_modules + target: /workspaces/speckle-server/packages/frontend-2/node_modules + read_only: false + - type: volume + source: monitor-deployment-node_modules + target: /workspaces/speckle-server/packages/monitor-deployment/node_modules + read_only: false + - type: volume + source: objectloader-node_modules + target: /workspaces/speckle-server/packages/objectloader/node_modules + read_only: false + - type: volume + source: objectsender-node_modules + target: /workspaces/speckle-server/packages/objectsender/node_modules + read_only: false + - type: volume + source: preview-frontend-node_modules + target: /workspaces/speckle-server/packages/preview-frontend/node_modules + read_only: false + - type: volume + source: preview-service-node_modules + target: /workspaces/speckle-server/packages/preview-service/node_modules + read_only: false + - type: volume + source: server-node_modules + target: /workspaces/speckle-server/packages/server/node_modules + read_only: false + - type: volume + source: shared-node_modules + target: /workspaces/speckle-server/packages/shared/node_modules + read_only: false + - type: volume + source: tailwind-theme-node_modules + target: /workspaces/speckle-server/packages/tailwind-theme/node_modules + read_only: false + - type: volume + source: ui-components-node_modules + target: /workspaces/speckle-server/packages/ui-components/node_modules + read_only: false + - type: volume + source: ui-components-nuxt-node_modules + target: /workspaces/speckle-server/packages/ui-components-nuxt/node_modules + read_only: false + - type: volume + source: viewer-node_modules + target: /workspaces/speckle-server/packages/viewer/node_modules + read_only: false + - type: volume + source: viewer-sandbox-node_modules + target: /workspaces/speckle-server/packages/viewer-sandbox/node_modules + read_only: false + - type: volume + source: webhook-service-node_modules + target: /workspaces/speckle-server/packages/webhook-service/node_modules + read_only: false + command: sleep infinity + depends_on: + - init + # 'host' network_mode makes services provided via docker-compose-deps available at 127.0.0.1 or localhost. + # They are not available via Docker Compose DNS resolution of the service name e.g. `postgres`. + # This is to ensure that the .env file used for local development remains compatible when running in devcontainer + network_mode: host + + # The mcr.microsoft.com/devcontainers/typescript-node docker image used for devcontainer + # runs as user 'node' and not as 'root' + # The mounted volumes (for the node_module CPU architecture build hack) are owned by 'root' by default + # This container will chown these node_modules volumes to the 'node' user + init: + image: debian:bookworm-slim + restart: 'no' + entrypoint: | + /bin/bash -c "groupadd --gid 1000 node && useradd --uid 1000 --gid node --shell /bin/bash --create-home node && find . -type d -name 'node_modules' | grep '\/node_modules$' | xargs chown node:node {}" + volumes: + # We need to override all node_modules directories + # by mounting an empty volume to avoid conflicting + # CPU build instructions between the host machine and the DevContainer's docker image + # (this allows the devcontainer to be based on linux yet work on Apple Silicon etc..) + # If you add a new package with a new `node_modules`, it needs to be added here + - type: volume + source: node_modules + target: /workspaces/speckle-server/node_modules + read_only: false + - type: volume + source: dui3-node_modules + target: /workspaces/speckle-server/packages/dui3/node_modules + read_only: false + - type: volume + source: fileimport-service-node_modules + target: /workspaces/speckle-server/packages/fileimport-service/node_modules + read_only: false + - type: volume + source: frontend-2-node_modules + target: /workspaces/speckle-server/packages/frontend-2/node_modules + read_only: false + - type: volume + source: monitor-deployment-node_modules + target: /workspaces/speckle-server/packages/monitor-deployment/node_modules + read_only: false + - type: volume + source: objectloader-node_modules + target: /workspaces/speckle-server/packages/objectloader/node_modules + read_only: false + - type: volume + source: objectsender-node_modules + target: /workspaces/speckle-server/packages/objectsender/node_modules + read_only: false + - type: volume + source: preview-frontend-node_modules + target: /workspaces/speckle-server/packages/preview-frontend/node_modules + read_only: false + - type: volume + source: preview-service-node_modules + target: /workspaces/speckle-server/packages/preview-service/node_modules + read_only: false + - type: volume + source: server-node_modules + target: /workspaces/speckle-server/packages/server/node_modules + read_only: false + - type: volume + source: shared-node_modules + target: /workspaces/speckle-server/packages/shared/node_modules + read_only: false + - type: volume + source: tailwind-theme-node_modules + target: /workspaces/speckle-server/packages/tailwind-theme/node_modules + read_only: false + - type: volume + source: ui-components-node_modules + target: /workspaces/speckle-server/packages/ui-components/node_modules + read_only: false + - type: volume + source: ui-components-nuxt-node_modules + target: /workspaces/speckle-server/packages/ui-components-nuxt/node_modules + read_only: false + - type: volume + source: viewer-node_modules + target: /workspaces/speckle-server/packages/viewer/node_modules + read_only: false + - type: volume + source: viewer-sandbox-node_modules + target: /workspaces/speckle-server/packages/viewer-sandbox/node_modules + read_only: false + - type: volume + source: webhook-service-node_modules + target: /workspaces/speckle-server/packages/webhook-service/node_modules + read_only: false + +volumes: + # We need to override all node_modules directories + # by mounting an empty volume to avoid conflicting + # CPU build instructions between the host machine and the DevContainer's docker image + # (this allows the devcontainer to be based on linux yet work on Apple Silicon etc..) + # If you add a new package with a new `node_modules`, it needs to be added here + node_modules: + dui3-node_modules: + fileimport-service-node_modules: + frontend-2-node_modules: + monitor-deployment-node_modules: + objectloader-node_modules: + objectsender-node_modules: + preview-frontend-node_modules: + preview-service-node_modules: + server-node_modules: + shared-node_modules: + tailwind-theme-node_modules: + ui-components-node_modules: + ui-components-nuxt-node_modules: + viewer-node_modules: + viewer-sandbox-node_modules: + webhook-service-node_modules: diff --git a/docker-compose-deps.yml b/docker-compose-deps.yml index e6e59ea02..899019887 100644 --- a/docker-compose-deps.yml +++ b/docker-compose-deps.yml @@ -1,4 +1,3 @@ -version: '3' services: # Actual Speckle Server dependencies diff --git a/packages/frontend-2/lib/core/configs/apollo.ts b/packages/frontend-2/lib/core/configs/apollo.ts index 4950f3eb4..0b6d344ce 100644 --- a/packages/frontend-2/lib/core/configs/apollo.ts +++ b/packages/frontend-2/lib/core/configs/apollo.ts @@ -382,6 +382,7 @@ function createLink(params: { // only log as error if at least one error has a status code of 5xx or has no status code const shouldLogAsWarn = gqlErrors.every( (e) => + e.extensions && 'statusCode' in e.extensions && typeof e.extensions.statusCode === 'number' && e.extensions.statusCode < 500 diff --git a/packages/frontend-2/lib/core/helpers/observability.ts b/packages/frontend-2/lib/core/helpers/observability.ts index be9a4cd04..aa4ee3f39 100644 --- a/packages/frontend-2/lib/core/helpers/observability.ts +++ b/packages/frontend-2/lib/core/helpers/observability.ts @@ -104,6 +104,12 @@ export const formatAppError = (err: SimpleError): SimpleError => { finalStatusCode = 429 } + if (finalMessage.match(/\/_nuxt\/builds\/meta.*?404/i)) { + finalMessage = + 'Speckle is currently upgrading to a newer version. Please reload the page in a few seconds.' + finalStatusCode = 500 + } + finalMessage = upperFirst(finalMessage) return { @@ -165,7 +171,7 @@ export function enableCustomLoggerHandling(params: { }) .filter((arg) => { // Filter out falsy values - return !!arg + return !!arg && !(['null', 'undefined'] as unknown[]).includes(arg) }) // If nothing valid to log, skip entirely diff --git a/packages/frontend-2/middleware/002-redirects.global.ts b/packages/frontend-2/middleware/002-redirects.global.ts index 28280d5e5..ffd0d3cf8 100644 --- a/packages/frontend-2/middleware/002-redirects.global.ts +++ b/packages/frontend-2/middleware/002-redirects.global.ts @@ -73,7 +73,7 @@ export default defineNuxtRouteMiddleware(async (to) => { const apollo = useApolloClientFromNuxt() const resourceBuilder = () => SpeckleViewer.ViewerRoute.resourceBuilder() - if (['/streams', '/commits'].includes(path)) { + if (['/streams', '/commits', '/streams/', '/commits/'].includes(path)) { return navigateTo(homeRoute) } diff --git a/packages/frontend-2/nuxt.config.ts b/packages/frontend-2/nuxt.config.ts index 9b2203f46..b6360c6f8 100644 --- a/packages/frontend-2/nuxt.config.ts +++ b/packages/frontend-2/nuxt.config.ts @@ -74,6 +74,10 @@ export default defineNuxtConfig({ } }, + experimental: { + emitRouteChunkError: 'automatic-immediate' + }, + alias: { // Rewriting all lodash calls to lodash-es for proper tree-shaking & chunk splitting // lodash: 'lodash-es' diff --git a/packages/frontend-2/plugins/001-logger.ts b/packages/frontend-2/plugins/001-logger.ts index 270e873fe..ba624ac53 100644 --- a/packages/frontend-2/plugins/001-logger.ts +++ b/packages/frontend-2/plugins/001-logger.ts @@ -324,7 +324,8 @@ export default defineNuxtPlugin(async (nuxtApp) => { if (!import.meta.server) { nuxtApp.hook('app:mounted', () => { logger.info('App mounted in the client', { - important: true + important: true, + speckleServerVersion }) }) } diff --git a/packages/server/.mocharc.js b/packages/server/.mocharc.js index 20fda1bd7..354f73cd5 100644 --- a/packages/server/.mocharc.js +++ b/packages/server/.mocharc.js @@ -13,7 +13,7 @@ const ignore = [ /** @type {import("mocha").MochaOptions} */ const config = { - spec: ['modules/**/*.spec.js', 'modules/**/*.spec.ts', 'logging/**/*.spec.ts'], + spec: ['modules/**/*.spec.js', 'modules/**/*.spec.ts', 'observability/**/*.spec.ts'], require: ['ts-node/register', 'test/hooks.ts'], ...(ignore.length ? { ignore } : {}), slow: 0, diff --git a/packages/server/app.ts b/packages/server/app.ts index 4a79f20a9..d65e81681 100644 --- a/packages/server/app.ts +++ b/packages/server/app.ts @@ -1,6 +1,6 @@ /* eslint-disable camelcase */ -/* eslint-disable no-restricted-imports */ /* istanbul ignore file */ +// eslint-disable-next-line no-restricted-imports import './bootstrap' import http from 'http' import express, { Express } from 'express' @@ -10,20 +10,20 @@ import 'express-async-errors' import cookieParser from 'cookie-parser' import { createTerminus } from '@godaddy/terminus' -import Metrics from '@/logging' +import Metrics from '@/observability' import { startupLogger, shutdownLogger, subscriptionLogger, graphqlLogger -} from '@/logging/logging' +} from '@/observability/logging' import { DetermineRequestIdMiddleware, LoggingExpressMiddleware, sanitizeHeaders -} from '@/logging/expressLogging' +} from '@/observability/components/express/expressLogging' -import { errorMetricsMiddleware } from '@/logging/errorMetrics' +import { errorMetricsMiddleware } from '@/observability/components/express/metrics/errorMetrics' import prometheusClient from 'prom-client' import { ApolloServer } from '@apollo/server' @@ -36,7 +36,7 @@ import { SubscriptionServer } from 'subscriptions-transport-ws' import { execute, subscribe } from 'graphql' import knex, { db } from '@/db/knex' -import { monitorActiveConnections } from '@/logging/httpServerMonitoring' +import { monitorActiveConnections } from '@/observability/components/httpServer/httpServerMonitoring' import { buildErrorFormatter } from '@/modules/core/graph/setup' import { getFileSizeLimitMB, @@ -82,10 +82,10 @@ import { enterNewRequestContext, getRequestContext, initiateRequestContextMiddleware -} from '@/logging/requestContext' +} from '@/observability/components/express/requestContext' import { randomUUID } from 'crypto' -import { onOperationHandlerFactory } from '@/logging/apolloSubscriptions' -import { initApolloSubscriptionMonitoring } from './logging/apolloSubscriptionMonitoring' +import { onOperationHandlerFactory } from '@/observability/components/apollo/apolloSubscriptions' +import { initApolloSubscriptionMonitoring } from '@/observability/components/apollo/metrics/apolloSubscriptionMonitoring' import { createRateLimiterMiddleware } from '@/modules/core/rest/ratelimiter' const GRAPHQL_PATH = '/graphql' diff --git a/packages/server/bin/ts-www b/packages/server/bin/ts-www index 47e6c3604..c7a6b71d8 100755 --- a/packages/server/bin/ts-www +++ b/packages/server/bin/ts-www @@ -7,7 +7,7 @@ */ require('ts-node/register') -const { logger } = require('../logging/logging') +const { logger } = require('../observability/logging') const { init, startHttp } = require('../app') init() diff --git a/packages/server/bin/www b/packages/server/bin/www index d8744a143..7849628b6 100755 --- a/packages/server/bin/www +++ b/packages/server/bin/www @@ -1,7 +1,7 @@ #!/usr/bin/env node 'use strict' -const { logger } = require('../dist/logging/logging') +const { logger } = require('../dist/observability/logging') const { init, startHttp } = require('../dist/app') init() diff --git a/packages/server/bootstrap.js b/packages/server/bootstrap.js index 35fc7fc18..ead9317c7 100644 --- a/packages/server/bootstrap.js +++ b/packages/server/bootstrap.js @@ -23,8 +23,8 @@ const { isApolloMonitoringEnabled, getApolloServerVersion, getServerVersion -} = require('./modules/shared/helpers/envHelper') -const { logger } = require('@/logging/logging') +} = require('@/modules/shared/helpers/envHelper') +const { logger } = require('@/observability/logging') if (isApolloMonitoringEnabled() && !getApolloServerVersion()) { process.env.APOLLO_SERVER_USER_VERSION = getServerVersion() @@ -46,10 +46,10 @@ if (isTestEnv()) { dotenv.config({ path: `${packageRoot}/.env` }) // knex is a singleton controlled by module so can't wait til app init -const { initOpenTelemetry } = require('./otel') +const { initOpenTelemetry } = require('@/observability/otel') initOpenTelemetry() -const { patchKnex } = require('./modules/core/patches/knex') +const { patchKnex } = require('@/modules/core/patches/knex') patchKnex() module.exports = { diff --git a/packages/server/db/knex.ts b/packages/server/db/knex.ts index 4c651f584..fc250f45a 100644 --- a/packages/server/db/knex.ts +++ b/packages/server/db/knex.ts @@ -1,7 +1,7 @@ /* istanbul ignore file */ const env = process.env.NODE_ENV || 'development' import configs from '@/knexfile' -import { dbStartupLogger } from '@/logging/logging' +import { dbStartupLogger } from '@/observability/logging' import knex from 'knex' const config = configs[env] diff --git a/packages/server/db/migrations.ts b/packages/server/db/migrations.ts index fee45792f..7f462fa33 100644 --- a/packages/server/db/migrations.ts +++ b/packages/server/db/migrations.ts @@ -1,7 +1,7 @@ import { Knex } from 'knex' import { DatabaseError } from '@/modules/shared/errors' import { ensureError } from '@speckle/shared' -import { logger } from '@/logging/logging' +import { startupLogger } from '@/observability/logging' export const migrateDbToLatest = async (params: { db: Knex; region: string }) => { const { db, region } = params @@ -11,7 +11,7 @@ export const migrateDbToLatest = async (params: { db: Knex; region: string }) => await db.migrate.latest() const durationMs = endStopWatch().milliseconds - logger.info( + startupLogger.info( { region, durationMs }, 'Migrated db to latest for region "{region}" in {durationMs}ms.' ) diff --git a/packages/server/healthchecks/index.ts b/packages/server/healthchecks/index.ts index b4c7be301..4f935bdbf 100644 --- a/packages/server/healthchecks/index.ts +++ b/packages/server/healthchecks/index.ts @@ -1,4 +1,4 @@ -import { healthCheckLogger } from '@/logging/logging' +import { healthCheckLogger } from '@/observability/logging' import { highFrequencyMetricsCollectionPeriodMs } from '@/modules/shared/helpers/envHelper' import { handleLivenessFactory, handleReadinessFactory } from '@/healthchecks/health' import { FreeConnectionsCalculator, ReadinessHandler } from '@/healthchecks/types' diff --git a/packages/server/knexfile.ts b/packages/server/knexfile.ts index b47137625..d5a86a1d6 100644 --- a/packages/server/knexfile.ts +++ b/packages/server/knexfile.ts @@ -12,7 +12,7 @@ import { postgresConnectionCreateTimeoutMillis, knexAsyncStackTracesEnabled } from '@/modules/shared/helpers/envHelper' -import { dbLogger as logger } from '@/logging/logging' +import { dbLogger as logger } from '@/observability/logging' import { Knex } from 'knex' import { createKnexConfig, diff --git a/packages/server/modules/accessrequests/index.ts b/packages/server/modules/accessrequests/index.ts index 77ae0e9d7..65cdfa5ad 100644 --- a/packages/server/modules/accessrequests/index.ts +++ b/packages/server/modules/accessrequests/index.ts @@ -1,5 +1,5 @@ import { db } from '@/db/knex' -import { moduleLogger } from '@/logging/logging' +import { moduleLogger } from '@/observability/logging' import { initializeEventListenerFactory } from '@/modules/accessrequests/services/eventListener' import { getStreamCollaboratorsFactory } from '@/modules/core/repositories/streams' import { publishNotification } from '@/modules/notifications/services/publication' diff --git a/packages/server/modules/activitystream/index.ts b/packages/server/modules/activitystream/index.ts index 4647cb733..955ff9b6a 100644 --- a/packages/server/modules/activitystream/index.ts +++ b/packages/server/modules/activitystream/index.ts @@ -1,6 +1,6 @@ import { Optional, SpeckleModule } from '@/modules/shared/helpers/typeHelper' import { publishNotification } from '@/modules/notifications/services/publication' -import { activitiesLogger, moduleLogger } from '@/logging/logging' +import { activitiesLogger, moduleLogger } from '@/observability/logging' import { weeklyEmailDigestEnabled } from '@/modules/shared/helpers/envHelper' import { EventBus, getEventBus } from '@/modules/shared/services/eventBus' import { sendActivityNotificationsFactory } from '@/modules/activitystream/services/summary' diff --git a/packages/server/modules/apiexplorer/index.ts b/packages/server/modules/apiexplorer/index.ts index 07a3088b5..9142961c9 100644 --- a/packages/server/modules/apiexplorer/index.ts +++ b/packages/server/modules/apiexplorer/index.ts @@ -1,5 +1,5 @@ import { SpeckleModule } from '@/modules/shared/helpers/typeHelper' -import { moduleLogger } from '@/logging/logging' +import { moduleLogger } from '@/observability/logging' import { readFile } from 'fs/promises' import { getFrontendOrigin } from '@/modules/shared/helpers/envHelper' diff --git a/packages/server/modules/auth/index.ts b/packages/server/modules/auth/index.ts index 3cba6323e..9d2eb36a0 100644 --- a/packages/server/modules/auth/index.ts +++ b/packages/server/modules/auth/index.ts @@ -1,7 +1,7 @@ import { SpeckleModule } from '@/modules/shared/helpers/typeHelper' import { registerOrUpdateScopeFactory } from '@/modules/shared/repositories/scopes' -import { authLogger, moduleLogger } from '@/logging/logging' +import { authLogger, moduleLogger } from '@/observability/logging' import db from '@/db/knex' import { initializeDefaultAppsFactory } from '@/modules/auth/services/serverApps' import { diff --git a/packages/server/modules/auth/repositories/apps.ts b/packages/server/modules/auth/repositories/apps.ts index 2ebbe6fd8..3008a904b 100644 --- a/packages/server/modules/auth/repositories/apps.ts +++ b/packages/server/modules/auth/repositories/apps.ts @@ -1,4 +1,4 @@ -import { logger, moduleLogger } from '@/logging/logging' +import { logger, moduleLogger } from '@/observability/logging' import { getDefaultApp } from '@/modules/auth/defaultApps' import { CreateApp, diff --git a/packages/server/modules/auth/services/postAuth.ts b/packages/server/modules/auth/services/postAuth.ts index 36d5b4ea2..d903e1013 100644 --- a/packages/server/modules/auth/services/postAuth.ts +++ b/packages/server/modules/auth/services/postAuth.ts @@ -1,5 +1,5 @@ -import { authLogger, type Logger } from '@/logging/logging' -import { maybeLoggerWithContext } from '@/logging/requestContext' +import { authLogger, type Logger } from '@/observability/logging' +import { maybeLoggerWithContext } from '@/observability/components/express/requestContext' import { addToMailchimpAudience, triggerMailchimpCustomerJourney diff --git a/packages/server/modules/auth/tests/auth.spec.ts b/packages/server/modules/auth/tests/auth.spec.ts index 13c4aced6..896dd5056 100644 --- a/packages/server/modules/auth/tests/auth.spec.ts +++ b/packages/server/modules/auth/tests/auth.spec.ts @@ -56,7 +56,8 @@ import { RateLimiterMemory } from 'rate-limiter-flexible' import { TIME } from '@speckle/shared' import type { Application } from 'express' import { passportAuthenticationCallbackFactory } from '@/modules/auth/services/passportService' -import { testLogger as logger } from '@/logging/logging' +import { testLogger as logger } from '@/observability/logging' +import { Application } from 'express' const getServerInfo = getServerInfoFactory({ db }) const getUser = getUserFactory({ db }) diff --git a/packages/server/modules/automate/clients/executionEngine.ts b/packages/server/modules/automate/clients/executionEngine.ts index 2c7eac46a..c60cdd908 100644 --- a/packages/server/modules/automate/clients/executionEngine.ts +++ b/packages/server/modules/automate/clients/executionEngine.ts @@ -1,4 +1,4 @@ -import { automateLogger } from '@/logging/logging' +import { automateLogger } from '@/observability/logging' import { ExecutionEngineBadResponseBodyError, type ExecutionEngineErrorResponse, diff --git a/packages/server/modules/automate/index.ts b/packages/server/modules/automate/index.ts index 6d0d5f904..5ed5c1bb9 100644 --- a/packages/server/modules/automate/index.ts +++ b/packages/server/modules/automate/index.ts @@ -1,4 +1,4 @@ -import { automateLogger, moduleLogger } from '@/logging/logging' +import { automateLogger, moduleLogger } from '@/observability/logging' import { Optional, SpeckleModule } from '@/modules/shared/helpers/typeHelper' import { onModelVersionCreateFactory, @@ -58,7 +58,7 @@ import { getEventBus } from '@/modules/shared/services/eventBus' import { VersionEvents } from '@/modules/core/domain/commits/events' import { AutomationEvents, AutomationRunEvents } from '@/modules/automate/domain/events' import { LogicError } from '@/modules/shared/errors' -import { maybeLoggerWithContext } from '@/logging/requestContext' +import { maybeLoggerWithContext } from '@/observability/components/express/requestContext' const { FF_AUTOMATE_MODULE_ENABLED } = getFeatureFlags() let quitListeners: Optional<() => void> = undefined diff --git a/packages/server/modules/automate/services/authCode.ts b/packages/server/modules/automate/services/authCode.ts index ab27afc41..ac2bf3233 100644 --- a/packages/server/modules/automate/services/authCode.ts +++ b/packages/server/modules/automate/services/authCode.ts @@ -1,4 +1,4 @@ -import { automateLogger } from '@/logging/logging' +import { automateLogger } from '@/observability/logging' import { CreateStoredAuthCode } from '@/modules/automate/domain/operations' import { AutomateAuthCodeHandshakeError } from '@/modules/automate/errors/management' import { EventBus } from '@/modules/shared/services/eventBus' diff --git a/packages/server/modules/automate/services/functionManagement.ts b/packages/server/modules/automate/services/functionManagement.ts index 564bfe9fa..45ff90edd 100644 --- a/packages/server/modules/automate/services/functionManagement.ts +++ b/packages/server/modules/automate/services/functionManagement.ts @@ -43,7 +43,7 @@ import { speckleAutomateUrl } from '@/modules/shared/helpers/envHelper' import { getFunctionsMarketplaceUrl } from '@/modules/core/helpers/routeHelper' -import { automateLogger } from '@/logging/logging' +import { automateLogger } from '@/observability/logging' import { CreateStoredAuthCode } from '@/modules/automate/domain/operations' import { GetUser } from '@/modules/core/domain/users/operations' import { noop } from 'lodash' diff --git a/packages/server/modules/automate/services/trigger.ts b/packages/server/modules/automate/services/trigger.ts index f40ff924c..a979875e2 100644 --- a/packages/server/modules/automate/services/trigger.ts +++ b/packages/server/modules/automate/services/trigger.ts @@ -25,7 +25,7 @@ import { import { TriggerAutomationError } from '@/modules/automate/errors/runs' import { ContextResourceAccessRules } from '@/modules/core/helpers/token' import { TokenResourceIdentifierType } from '@/modules/core/graph/generated/graphql' -import { automateLogger } from '@/logging/logging' +import { automateLogger } from '@/observability/logging' import { FunctionInputDecryptor } from '@/modules/automate/services/encryption' import { LibsodiumEncryptionError } from '@/modules/shared/errors/encryption' import { diff --git a/packages/server/modules/blobstorage/index.ts b/packages/server/modules/blobstorage/index.ts index 112fe1edc..fb6e05ca7 100644 --- a/packages/server/modules/blobstorage/index.ts +++ b/packages/server/modules/blobstorage/index.ts @@ -15,7 +15,7 @@ import { ResourceMismatch, BadRequestError } from '@/modules/shared/errors' -import { moduleLogger, logger } from '@/logging/logging' +import { moduleLogger } from '@/observability/logging' import { getAllStreamBlobIdsFactory, upsertBlobFactory, @@ -68,7 +68,7 @@ const ensureConditions = async () => { } if (!process.env.S3_BUCKET) { - logger.warn( + moduleLogger.warn( 'S3_BUCKET env variable was not specified. 📦 BlobStorage will be DISABLED.' ) return diff --git a/packages/server/modules/cli/commands/activities/send.ts b/packages/server/modules/cli/commands/activities/send.ts index 887b1be5e..e06e71b72 100644 --- a/packages/server/modules/cli/commands/activities/send.ts +++ b/packages/server/modules/cli/commands/activities/send.ts @@ -1,7 +1,7 @@ import { CommandModule } from 'yargs' import { initializeQueue } from '@/modules/notifications/services/queue' import { publishNotification } from '@/modules/notifications/services/publication' -import { cliLogger } from '@/logging/logging' +import { cliLogger } from '@/observability/logging' import { sendActivityNotificationsFactory } from '@/modules/activitystream/services/summary' import { getActiveUserStreamsFactory } from '@/modules/activitystream/repositories' import { db } from '@/db/knex' diff --git a/packages/server/modules/cli/commands/bull/monitor.ts b/packages/server/modules/cli/commands/bull/monitor.ts index fc684ba12..7501d2ebb 100644 --- a/packages/server/modules/cli/commands/bull/monitor.ts +++ b/packages/server/modules/cli/commands/bull/monitor.ts @@ -8,7 +8,7 @@ import { buildNotificationsQueue } from '@/modules/notifications/services/queue' import { noop } from 'lodash' -import { cliLogger } from '@/logging/logging' +import { cliLogger } from '@/observability/logging' const PORT = 3032 diff --git a/packages/server/modules/cli/commands/bull/test-consume.ts b/packages/server/modules/cli/commands/bull/test-consume.ts index 3db0e8b25..867b41033 100644 --- a/packages/server/modules/cli/commands/bull/test-consume.ts +++ b/packages/server/modules/cli/commands/bull/test-consume.ts @@ -1,4 +1,4 @@ -import { cliLogger } from '@/logging/logging' +import { cliLogger } from '@/observability/logging' import { NotificationType } from '@/modules/notifications/helpers/types' import { initializeConsumption } from '@/modules/notifications/index' import { EnvironmentResourceError } from '@/modules/shared/errors' diff --git a/packages/server/modules/cli/commands/bull/test-push.ts b/packages/server/modules/cli/commands/bull/test-push.ts index c1f9bbd42..9e566275d 100644 --- a/packages/server/modules/cli/commands/bull/test-push.ts +++ b/packages/server/modules/cli/commands/bull/test-push.ts @@ -1,4 +1,4 @@ -import { cliLogger } from '@/logging/logging' +import { cliLogger } from '@/observability/logging' import { MentionedInCommentData, NotificationType diff --git a/packages/server/modules/cli/commands/db/migrate/create.ts b/packages/server/modules/cli/commands/db/migrate/create.ts index f2b365b5a..d790db3ed 100644 --- a/packages/server/modules/cli/commands/db/migrate/create.ts +++ b/packages/server/modules/cli/commands/db/migrate/create.ts @@ -1,7 +1,7 @@ import knex from '@/db/knex' import { appRoot } from '@/bootstrap' import fs from 'fs/promises' -import { logger } from '@/logging/logging' +import { cliLogger as logger } from '@/observability/logging' import { CommandModule } from 'yargs' import { ensureError } from '@speckle/shared' import { MisconfiguredEnvironmentError } from '@/modules/shared/errors' diff --git a/packages/server/modules/cli/commands/db/migrate/down.ts b/packages/server/modules/cli/commands/db/migrate/down.ts index c7ba1aef4..3ccb19add 100644 --- a/packages/server/modules/cli/commands/db/migrate/down.ts +++ b/packages/server/modules/cli/commands/db/migrate/down.ts @@ -1,4 +1,4 @@ -import { logger } from '@/logging/logging' +import { cliLogger as logger } from '@/observability/logging' import { CommonDbArgs, getTargettedDbClients } from '@/modules/cli/commands/db/helpers' import { CommandModule } from 'yargs' diff --git a/packages/server/modules/cli/commands/db/migrate/latest.ts b/packages/server/modules/cli/commands/db/migrate/latest.ts index 0fec082a6..d2be438f5 100644 --- a/packages/server/modules/cli/commands/db/migrate/latest.ts +++ b/packages/server/modules/cli/commands/db/migrate/latest.ts @@ -1,4 +1,4 @@ -import { logger } from '@/logging/logging' +import { cliLogger as logger } from '@/observability/logging' import { CommonDbArgs, getTargettedDbClients } from '@/modules/cli/commands/db/helpers' import { CommandModule } from 'yargs' diff --git a/packages/server/modules/cli/commands/db/migrate/rollback.ts b/packages/server/modules/cli/commands/db/migrate/rollback.ts index 4e1c49edf..eefee8584 100644 --- a/packages/server/modules/cli/commands/db/migrate/rollback.ts +++ b/packages/server/modules/cli/commands/db/migrate/rollback.ts @@ -1,4 +1,4 @@ -import { logger } from '@/logging/logging' +import { cliLogger as logger } from '@/observability/logging' import { CommonDbArgs, getTargettedDbClients } from '@/modules/cli/commands/db/helpers' import { resetPubSubFactory } from '@/test/hooks' import { CommandModule } from 'yargs' diff --git a/packages/server/modules/cli/commands/db/migrate/up.ts b/packages/server/modules/cli/commands/db/migrate/up.ts index 0e33db39f..6fd159923 100644 --- a/packages/server/modules/cli/commands/db/migrate/up.ts +++ b/packages/server/modules/cli/commands/db/migrate/up.ts @@ -1,4 +1,4 @@ -import { logger } from '@/logging/logging' +import { cliLogger as logger } from '@/observability/logging' import { CommonDbArgs, getTargettedDbClients } from '@/modules/cli/commands/db/helpers' import { CommandModule } from 'yargs' diff --git a/packages/server/modules/cli/commands/db/seed/commits.ts b/packages/server/modules/cli/commands/db/seed/commits.ts index 77247f082..5e01cfc5e 100644 --- a/packages/server/modules/cli/commands/db/seed/commits.ts +++ b/packages/server/modules/cli/commands/db/seed/commits.ts @@ -1,5 +1,5 @@ import { db } from '@/db/knex' -import { cliLogger } from '@/logging/logging' +import { cliLogger as logger } from '@/observability/logging' import { StreamNotFoundError } from '@/modules/core/errors/stream' import { UserNotFoundError } from '@/modules/core/errors/user' import { getStreamFactory } from '@/modules/core/repositories/streams' @@ -55,7 +55,7 @@ const command: CommandModule< ) } - cliLogger.info(`Generating ${count} objects & commits for stream ${streamId}...`) + logger.info(`Generating ${count} objects & commits for stream ${streamId}...`) await createTestCommits( times( count, @@ -68,7 +68,7 @@ const command: CommandModule< }) ) ) - cliLogger.info(`...done`) + logger.info(`...done`) } } diff --git a/packages/server/modules/cli/commands/db/seed/users.ts b/packages/server/modules/cli/commands/db/seed/users.ts index 05bba7187..023bbed16 100644 --- a/packages/server/modules/cli/commands/db/seed/users.ts +++ b/packages/server/modules/cli/commands/db/seed/users.ts @@ -1,4 +1,4 @@ -import { logger } from '@/logging/logging' +import { cliLogger as logger } from '@/observability/logging' import { Users, ServerAcl } from '@/modules/core/dbSchema' import { Roles } from '@/modules/core/helpers/mainConstants' import { faker } from '@faker-js/faker' diff --git a/packages/server/modules/cli/commands/download/commit.ts b/packages/server/modules/cli/commands/download/commit.ts index e04394ae5..38b20a8d2 100644 --- a/packages/server/modules/cli/commands/download/commit.ts +++ b/packages/server/modules/cli/commands/download/commit.ts @@ -1,6 +1,6 @@ import { CommandModule } from 'yargs' import { downloadCommitFactory } from '@/modules/cross-server-sync/services/commit' -import { cliLogger } from '@/logging/logging' +import { cliLogger as logger } from '@/observability/logging' import { getStreamCollaboratorsFactory, getStreamFactory, @@ -181,7 +181,7 @@ const command: CommandModule< createCommentReplyAndNotify }) - await downloadCommit(argv, { logger: cliLogger }) + await downloadCommit(argv, { logger }) } } diff --git a/packages/server/modules/cli/commands/download/project.ts b/packages/server/modules/cli/commands/download/project.ts index 8faf71097..ea43edcd3 100644 --- a/packages/server/modules/cli/commands/download/project.ts +++ b/packages/server/modules/cli/commands/download/project.ts @@ -1,5 +1,5 @@ import { CommandModule } from 'yargs' -import { cliLogger } from '@/logging/logging' +import { cliLogger as logger } from '@/observability/logging' import { downloadProjectFactory } from '@/modules/cross-server-sync/services/project' import { downloadCommitFactory } from '@/modules/cross-server-sync/services/commit' import { @@ -226,7 +226,7 @@ const command: CommandModule< eventEmit: getEventBus().emit }) }) - await downloadProject({ ...argv, regionKey }, { logger: cliLogger }) + await downloadProject({ ...argv, regionKey }, { logger }) } } diff --git a/packages/server/modules/cli/commands/graphql/introspect.ts b/packages/server/modules/cli/commands/graphql/introspect.ts index e2dbfa61b..26b57b742 100644 --- a/packages/server/modules/cli/commands/graphql/introspect.ts +++ b/packages/server/modules/cli/commands/graphql/introspect.ts @@ -1,5 +1,5 @@ import { CommandModule } from 'yargs' -import { cliLogger } from '@/logging/logging' +import { cliLogger as logger } from '@/observability/logging' import * as ModulesSetup from '@/modules' import { printSchema } from 'graphql/utilities' import fs from 'node:fs/promises' @@ -17,11 +17,11 @@ const command: CommandModule = { } }, handler: async ({ file }) => { - cliLogger.info('Loading GQL schema...') + logger.info('Loading GQL schema...') const schema = ModulesSetup.graphSchema() const schemaString = printSchema(schema) - cliLogger.info(`Saving to "${file}"...`) + logger.info(`Saving to "${file}"...`) const absolutePath = path.isAbsolute(file) ? file : path.resolve(process.cwd(), file) diff --git a/packages/server/modules/cli/commands/stream/clone.ts b/packages/server/modules/cli/commands/stream/clone.ts index 901a10ffb..a5eefc0bf 100644 --- a/packages/server/modules/cli/commands/stream/clone.ts +++ b/packages/server/modules/cli/commands/stream/clone.ts @@ -1,5 +1,5 @@ import { db } from '@/db/knex' -import { cliLogger } from '@/logging/logging' +import { cliLogger as logger } from '@/observability/logging' import { getBatchedStreamCommentsFactory, getCommentLinksFactory, @@ -66,11 +66,11 @@ const command: CommandModule< emitEvent: getEventBus().emit }) - cliLogger.info( + logger.info( `Cloning stream ${sourceStreamId} into the account of user ${targetUserId}...` ) const { id } = await cloneStream(targetUserId, sourceStreamId) - cliLogger.info('Cloning successful! New stream ID: ' + id) + logger.info('Cloning successful! New stream ID: ' + id) } } diff --git a/packages/server/modules/cli/commands/test/test-meta.ts b/packages/server/modules/cli/commands/test/test-meta.ts index 39832b853..7c3c7a13e 100644 --- a/packages/server/modules/cli/commands/test/test-meta.ts +++ b/packages/server/modules/cli/commands/test/test-meta.ts @@ -1,5 +1,5 @@ import { CommandModule } from 'yargs' -import { cliLogger } from '@/logging/logging' +import { cliLogger as logger } from '@/observability/logging' import { metaHelpers } from '@/modules/core/helpers/meta' import { Users } from '@/modules/core/dbSchema' import { UserRecord, UsersMetaRecord } from '@/modules/core/helpers/types' @@ -9,28 +9,28 @@ const command: CommandModule = { command: 'test-meta', describe: 'Testing users meta behaviour', handler: async () => { - cliLogger.info('Hello world!') + logger.info('Hello world!') const testUsers = await Users.knex().limit(5) const firstUserId = testUsers[0]?.id const secondUserId = testUsers[1]?.id if (!firstUserId || !secondUserId) { - cliLogger.error('One or more test users were not found') + logger.error('One or more test users were not found') return } const meta = metaHelpers(Users, db) // set value - cliLogger.info(await meta.set(firstUserId, 'foo', false)) - cliLogger.info(await meta.set(firstUserId, 'bar', "I'm happy to see ya brodie'!\"")) - cliLogger.info(await meta.set(secondUserId, 'foo', { a: 123 })) + logger.info(await meta.set(firstUserId, 'foo', false)) + logger.info(await meta.set(firstUserId, 'bar', "I'm happy to see ya brodie'!\"")) + logger.info(await meta.set(secondUserId, 'foo', { a: 123 })) // get value - cliLogger.info(await meta.get>(firstUserId, 'bar')) + logger.info(await meta.get>(firstUserId, 'bar')) // get multiple values - cliLogger.info( + logger.info( await meta.getMultiple([ { id: firstUserId, key: 'foo' }, { id: firstUserId, key: 'bar' }, @@ -40,7 +40,7 @@ const command: CommandModule = { ) // delete value - cliLogger.info(await meta.delete(firstUserId, 'bar')) + logger.info(await meta.delete(firstUserId, 'bar')) } } diff --git a/packages/server/modules/cli/commands/workspaces/set-plan.ts b/packages/server/modules/cli/commands/workspaces/set-plan.ts index 6bda574f4..6a644ccd8 100644 --- a/packages/server/modules/cli/commands/workspaces/set-plan.ts +++ b/packages/server/modules/cli/commands/workspaces/set-plan.ts @@ -1,5 +1,5 @@ import { CommandModule } from 'yargs' -import { cliLogger } from '@/logging/logging' +import { cliLogger as logger } from '@/observability/logging' import { getWorkspaceBySlugOrIdFactory } from '@/modules/workspaces/repositories/workspaces' import { db } from '@/db/knex' import { upsertPaidWorkspacePlanFactory } from '@/modules/gatekeeper/repositories/billing' @@ -42,7 +42,7 @@ const command: CommandModule< } }, handler: async (args) => { - cliLogger.info( + logger.info( `Setting plan for workspace '${args.workspaceSlugOrId}' to '${args.plan}' with status '${args.status}'` ) const workspace = await getWorkspaceBySlugOrIdFactory({ db })(args) @@ -60,7 +60,7 @@ const command: CommandModule< status: args.status } }) - cliLogger.info(`Plan set!`) + logger.info(`Plan set!`) } } diff --git a/packages/server/modules/cli/index.ts b/packages/server/modules/cli/index.ts index 7f6d64620..1198e18df 100644 --- a/packages/server/modules/cli/index.ts +++ b/packages/server/modules/cli/index.ts @@ -2,7 +2,7 @@ import path from 'path' import yargs from 'yargs' import '../../bootstrap' -import { cliLogger, logger } from '@/logging/logging' +import { cliLogger as logger } from '@/observability/logging' import { isTestEnv } from '@/modules/shared/helpers/envHelper' import { mochaHooks } from '@/test/hooks' @@ -23,7 +23,7 @@ const main = async () => { // In test env, run beforeAll hooks to properly initialize everything first if (isBeforeAllSet && isTestEnv()) { - cliLogger.info('Running test beforeAll hooks...') + logger.info('Running test beforeAll hooks...') await (mochaHooks.beforeAll as () => Promise)() } }) diff --git a/packages/server/modules/comments/index.ts b/packages/server/modules/comments/index.ts index bc5ea9d7a..a73bd1ebe 100644 --- a/packages/server/modules/comments/index.ts +++ b/packages/server/modules/comments/index.ts @@ -1,5 +1,5 @@ import { db } from '@/db/knex' -import { moduleLogger } from '@/logging/logging' +import { moduleLogger } from '@/observability/logging' import { saveActivityFactory } from '@/modules/activitystream/repositories' import { reportSubscriptionEventsFactory } from '@/modules/comments/events/subscriptionListeners' import { getCommentsResourcesFactory } from '@/modules/comments/repositories/comments' diff --git a/packages/server/modules/core/domain/objects/operations.ts b/packages/server/modules/core/domain/objects/operations.ts index cdd5927bc..9927256d8 100644 --- a/packages/server/modules/core/domain/objects/operations.ts +++ b/packages/server/modules/core/domain/objects/operations.ts @@ -1,4 +1,4 @@ -import { Logger } from '@/logging/logging' +import type { Logger } from '@/observability/logging' import { InsertableSpeckleObject, RawSpeckleObject, diff --git a/packages/server/modules/core/graph/plugins/logging.ts b/packages/server/modules/core/graph/plugins/logging.ts index af3faeaf9..f7c296dab 100644 --- a/packages/server/modules/core/graph/plugins/logging.ts +++ b/packages/server/modules/core/graph/plugins/logging.ts @@ -1,12 +1,12 @@ /* eslint-disable camelcase */ import { type Registry, Counter } from 'prom-client' -import { graphqlLogger } from '@/logging/logging' -import { redactSensitiveVariables } from '@/logging/loggingHelper' +import { graphqlLogger } from '@/observability/logging' +import { redactSensitiveVariables } from '@/observability/utils/redact' import { FieldNode, SelectionNode } from 'graphql' import { ApolloServerPlugin } from '@apollo/server' import { GraphQLContext } from '@/modules/shared/helpers/typeHelper' -import { shouldLogAsInfoLevel } from '@/logging/graphqlError' -import { getRequestContext } from '@/logging/requestContext' +import { shouldLogAsInfoLevel } from '@/observability/utils/logLevels' +import { getRequestContext } from '@/observability/components/express/requestContext' type ApolloLoggingPluginTransaction = { start: number diff --git a/packages/server/modules/core/graph/resolvers/users.ts b/packages/server/modules/core/graph/resolvers/users.ts index 6db82a25d..a7765363c 100644 --- a/packages/server/modules/core/graph/resolvers/users.ts +++ b/packages/server/modules/core/graph/resolvers/users.ts @@ -35,7 +35,7 @@ import { deleteStreamFactory, getUserDeletableStreamsFactory } from '@/modules/core/repositories/streams' -import { dbLogger } from '@/logging/logging' +import { dbLogger } from '@/observability/logging' import { getAdminUsersListCollectionFactory } from '@/modules/core/services/users/legacyAdminUsersList' import { Resolvers } from '@/modules/core/graph/generated/graphql' import { getServerInfoFactory } from '@/modules/core/repositories/server' diff --git a/packages/server/modules/core/index.ts b/packages/server/modules/core/index.ts index 195478e88..f96cc1b26 100644 --- a/packages/server/modules/core/index.ts +++ b/packages/server/modules/core/index.ts @@ -1,4 +1,4 @@ -import { moduleLogger } from '@/logging/logging' +import { moduleLogger } from '@/observability/logging' import { setupResultListener, shutdownResultListener diff --git a/packages/server/modules/core/logger.ts b/packages/server/modules/core/logger.ts index c7d5f7424..f55ca42c8 100644 --- a/packages/server/modules/core/logger.ts +++ b/packages/server/modules/core/logger.ts @@ -1,3 +1,3 @@ -import { extendLoggerComponent, moduleLogger } from '@/logging/logging' +import { extendLoggerComponent, moduleLogger } from '@/observability/logging' export const coreLogger = extendLoggerComponent(moduleLogger, 'core') diff --git a/packages/server/modules/core/rest/defaultErrorHandler.ts b/packages/server/modules/core/rest/defaultErrorHandler.ts index 7d8900433..9f46465b6 100644 --- a/packages/server/modules/core/rest/defaultErrorHandler.ts +++ b/packages/server/modules/core/rest/defaultErrorHandler.ts @@ -5,7 +5,7 @@ import { Optional, ensureError } from '@speckle/shared' import { ErrorRequestHandler } from 'express' import { get, isNumber } from 'lodash' import { VError } from 'verror' -import { logger as defaultLogger } from '@/logging/logging' +import { logger as defaultLogger } from '@/observability/logging' export const resolveStatusCode = (e: Error): number => { if (e instanceof BaseError) { diff --git a/packages/server/modules/core/rest/download.ts b/packages/server/modules/core/rest/download.ts index fc55e99c4..ca1733daa 100644 --- a/packages/server/modules/core/rest/download.ts +++ b/packages/server/modules/core/rest/download.ts @@ -3,7 +3,6 @@ import { corsMiddlewareFactory } from '@/modules/core/configs/cors' import { SpeckleObjectsStream } from '@/modules/core/rest/speckleObjectsStream' import { pipeline, PassThrough } from 'stream' -import { logger } from '@/logging/logging' import { getFormattedObjectFactory, getObjectChildrenStreamFactory @@ -25,7 +24,7 @@ export default (app: express.Express) => { app.options('/objects/:streamId/:objectId', corsMiddlewareFactory()) app.get('/objects/:streamId/:objectId', corsMiddlewareFactory(), async (req, res) => { - const boundLogger = (req.log || logger).child({ + const boundLogger = req.log.child({ requestId: req.id, userId: req.context.userId || '-', streamId: req.params.streamId, @@ -112,7 +111,7 @@ export default (app: express.Express) => { '/objects/:streamId/:objectId/single', corsMiddlewareFactory(), async (req, res) => { - const boundLogger = (req.log || logger).child({ + const boundLogger = req.log.child({ requestId: req.id, userId: req.context.userId || '-', streamId: req.params.streamId, diff --git a/packages/server/modules/core/services/objects/management.ts b/packages/server/modules/core/services/objects/management.ts index 2825f11b3..00ed3fd44 100644 --- a/packages/server/modules/core/services/objects/management.ts +++ b/packages/server/modules/core/services/objects/management.ts @@ -9,7 +9,7 @@ import { estimateStringMegabyteSize } from '@/modules/core/utils/chunking' import { ObjectHandlingError } from '@/modules/core/errors/object' -import { servicesLogger } from '@/logging/logging' +import { servicesLogger } from '@/observability/logging' import { CreateObject, CreateObjects, diff --git a/packages/server/modules/core/services/taskScheduler.ts b/packages/server/modules/core/services/taskScheduler.ts index 05c27e587..7649b5d48 100644 --- a/packages/server/modules/core/services/taskScheduler.ts +++ b/packages/server/modules/core/services/taskScheduler.ts @@ -1,6 +1,6 @@ import cron from 'node-cron' import { InvalidArgumentError } from '@/modules/shared/errors' -import { logger } from '@/logging/logging' +import { taskSchedulerLogger as logger } from '@/observability/logging' import { AcquireTaskLock, ReleaseTaskLock, diff --git a/packages/server/modules/core/services/users/management.ts b/packages/server/modules/core/services/users/management.ts index 65b9bc0a8..729539421 100644 --- a/packages/server/modules/core/services/users/management.ts +++ b/packages/server/modules/core/services/users/management.ts @@ -48,7 +48,7 @@ import { DeleteStreamRecord, GetUserDeletableStreams } from '@/modules/core/domain/streams/operations' -import { Logger } from '@/logging/logging' +import type { Logger } from '@/observability/logging' import { DeleteAllUserInvites } from '@/modules/serverinvites/domain/operations' import { GetServerInfo } from '@/modules/core/domain/server/operations' import { EventBusEmit } from '@/modules/shared/services/eventBus' diff --git a/packages/server/modules/core/tests/users.spec.ts b/packages/server/modules/core/tests/users.spec.ts index 1b00f7f1f..f2395bc8d 100644 --- a/packages/server/modules/core/tests/users.spec.ts +++ b/packages/server/modules/core/tests/users.spec.ts @@ -96,7 +96,7 @@ import { } from '@/modules/core/services/users/management' import { validateAndCreateUserEmailFactory } from '@/modules/core/services/userEmails' import { finalizeInvitedServerRegistrationFactory } from '@/modules/serverinvites/services/processing' -import { dbLogger } from '@/logging/logging' +import { dbLogger } from '@/observability/logging' import { storeApiTokenFactory, storeTokenScopesFactory, diff --git a/packages/server/modules/core/tests/usersAdmin.spec.ts b/packages/server/modules/core/tests/usersAdmin.spec.ts index 01c077365..2e81c506d 100644 --- a/packages/server/modules/core/tests/usersAdmin.spec.ts +++ b/packages/server/modules/core/tests/usersAdmin.spec.ts @@ -40,7 +40,7 @@ import { deleteStreamFactory, getUserDeletableStreamsFactory } from '@/modules/core/repositories/streams' -import { dbLogger } from '@/logging/logging' +import { dbLogger } from '@/observability/logging' import { getServerInfoFactory } from '@/modules/core/repositories/server' import { getEventBus } from '@/modules/shared/services/eventBus' import { expect } from 'chai' diff --git a/packages/server/modules/core/utils/dbNotificationListener.ts b/packages/server/modules/core/utils/dbNotificationListener.ts index 2691f193c..b140478df 100644 --- a/packages/server/modules/core/utils/dbNotificationListener.ts +++ b/packages/server/modules/core/utils/dbNotificationListener.ts @@ -1,5 +1,5 @@ import { MaybeAsync, Optional, md5, wait } from '@speckle/shared' -import { dbNotificationLogger } from '@/logging/logging' +import { dbNotificationLogger } from '@/observability/logging' import { knex } from '@/modules/core/dbSchema' import * as Knex from 'knex' import * as pg from 'pg' diff --git a/packages/server/modules/core/utils/ratelimiter.ts b/packages/server/modules/core/utils/ratelimiter.ts index d9e02b12f..d33eb5263 100644 --- a/packages/server/modules/core/utils/ratelimiter.ts +++ b/packages/server/modules/core/utils/ratelimiter.ts @@ -7,7 +7,7 @@ import { RateLimiterRes } from 'rate-limiter-flexible' import { Nullable, TIME } from '@speckle/shared' -import { rateLimiterLogger } from '@/logging/logging' +import { rateLimiterLogger } from '@/observability/logging' import { createRedisClient } from '@/modules/shared/redis/redis' import { RateLimitError } from '@/modules/core/errors/ratelimit' diff --git a/packages/server/modules/cross-server-sync/domain/operations.ts b/packages/server/modules/cross-server-sync/domain/operations.ts index 137578cd0..06a98c8d3 100644 --- a/packages/server/modules/cross-server-sync/domain/operations.ts +++ b/packages/server/modules/cross-server-sync/domain/operations.ts @@ -1,4 +1,4 @@ -import { type Logger } from '@/logging/logging' +import { type Logger } from '@/observability/logging' import { StreamRecord } from '@/modules/core/helpers/types' export type DownloadCommit = ( diff --git a/packages/server/modules/cross-server-sync/index.ts b/packages/server/modules/cross-server-sync/index.ts index 7e964e8f8..3866a4846 100644 --- a/packages/server/modules/cross-server-sync/index.ts +++ b/packages/server/modules/cross-server-sync/index.ts @@ -1,5 +1,5 @@ import { db } from '@/db/knex' -import { moduleLogger, crossServerSyncLogger } from '@/logging/logging' +import { moduleLogger, crossServerSyncLogger } from '@/observability/logging' import { getBlobsFactory } from '@/modules/blobstorage/repositories' import { getCommentFactory, diff --git a/packages/server/modules/cross-server-sync/services/commit.ts b/packages/server/modules/cross-server-sync/services/commit.ts index e85743da9..02710b2b2 100644 --- a/packages/server/modules/cross-server-sync/services/commit.ts +++ b/packages/server/modules/cross-server-sync/services/commit.ts @@ -5,7 +5,7 @@ import { CreateCommentInput } from '@/test/graphql/generated/graphql' import { Roles, timeoutAt } from '@speckle/shared' import ObjectLoader from '@speckle/objectloader' import { noop } from 'lodash' -import { crossServerSyncLogger } from '@/logging/logging' +import { crossServerSyncLogger } from '@/observability/logging' import type { SpeckleViewer } from '@speckle/shared' import { retry } from '@speckle/shared' import { diff --git a/packages/server/modules/cross-server-sync/services/onboardingProject.ts b/packages/server/modules/cross-server-sync/services/onboardingProject.ts index 352569d67..eb2414262 100644 --- a/packages/server/modules/cross-server-sync/services/onboardingProject.ts +++ b/packages/server/modules/cross-server-sync/services/onboardingProject.ts @@ -1,4 +1,4 @@ -import { crossServerSyncLogger } from '@/logging/logging' +import { crossServerSyncLogger } from '@/observability/logging' import { GetOnboardingBaseStream, MarkOnboardingBaseStream diff --git a/packages/server/modules/cross-server-sync/services/project.ts b/packages/server/modules/cross-server-sync/services/project.ts index d9ffb02e1..bcfd43f18 100644 --- a/packages/server/modules/cross-server-sync/services/project.ts +++ b/packages/server/modules/cross-server-sync/services/project.ts @@ -1,4 +1,4 @@ -import { crossServerSyncLogger, Logger } from '@/logging/logging' +import { crossServerSyncLogger, Logger } from '@/observability/logging' import { CrossServerProjectSyncError } from '@/modules/cross-server-sync/errors' import { createApolloClient, diff --git a/packages/server/modules/emails/index.ts b/packages/server/modules/emails/index.ts index c8929f0ae..f968aa0b9 100644 --- a/packages/server/modules/emails/index.ts +++ b/packages/server/modules/emails/index.ts @@ -1,5 +1,5 @@ /* istanbul ignore file */ -import { moduleLogger } from '@/logging/logging' +import { moduleLogger } from '@/observability/logging' import * as SendingService from '@/modules/emails/services/sending' import { initializeTransporter } from '@/modules/emails/utils/transporter' import { SpeckleModule } from '@/modules/shared/helpers/typeHelper' diff --git a/packages/server/modules/emails/services/sending.ts b/packages/server/modules/emails/services/sending.ts index 733b2b43a..ac5ee3d96 100644 --- a/packages/server/modules/emails/services/sending.ts +++ b/packages/server/modules/emails/services/sending.ts @@ -1,4 +1,4 @@ -import { logger } from '@/logging/logging' +import { emailLogger as logger } from '@/observability/logging' import { SendEmail, SendEmailParams } from '@/modules/emails/domain/operations' import { getTransporter } from '@/modules/emails/utils/transporter' import { getEmailFromAddress } from '@/modules/shared/helpers/envHelper' diff --git a/packages/server/modules/emails/utils/transporter.ts b/packages/server/modules/emails/utils/transporter.ts index 009c49d28..14b90b0df 100644 --- a/packages/server/modules/emails/utils/transporter.ts +++ b/packages/server/modules/emails/utils/transporter.ts @@ -1,4 +1,4 @@ -import { logger, moduleLogger } from '@/logging/logging' +import { emailLogger as logger } from '@/observability/logging' import { MisconfiguredEnvironmentError } from '@/modules/shared/errors' import { isEmailEnabled, isTestEnv } from '@/modules/shared/helpers/envHelper' import { createTransport, Transporter } from 'nodemailer' @@ -37,7 +37,7 @@ export async function initializeTransporter(): Promise if (!newTransporter) { const message = '📧 Email provider is enabled but transport has not initialized correctly. Please review the email configuration or your email system for problems.' - moduleLogger.error(message) + logger.error(message) throw new MisconfiguredEnvironmentError(message) } } @@ -47,13 +47,13 @@ export async function initializeTransporter(): Promise if (!newTransporter) { const message = '📧 In testing a mock email provider is enabled but transport has not initialized correctly.' - moduleLogger.error(message) + logger.error(message) throw new MisconfiguredEnvironmentError(message) } } if (!newTransporter) { - moduleLogger.warn( + logger.warn( '📧 Email provider is not configured. Server functionality will be limited.' ) } diff --git a/packages/server/modules/fileuploads/index.ts b/packages/server/modules/fileuploads/index.ts index 6c6d35c67..add682414 100644 --- a/packages/server/modules/fileuploads/index.ts +++ b/packages/server/modules/fileuploads/index.ts @@ -2,7 +2,7 @@ import { insertNewUploadAndNotifyFactory } from '@/modules/fileuploads/services/management' import request from 'request' import { authMiddlewareCreator } from '@/modules/shared/middleware' -import { moduleLogger } from '@/logging/logging' +import { moduleLogger } from '@/observability/logging' import { onFileImportProcessedFactory, onFileProcessingFactory, diff --git a/packages/server/modules/fileuploads/services/resultListener.ts b/packages/server/modules/fileuploads/services/resultListener.ts index 527e5ac9a..b26f57a82 100644 --- a/packages/server/modules/fileuploads/services/resultListener.ts +++ b/packages/server/modules/fileuploads/services/resultListener.ts @@ -12,7 +12,7 @@ import { GetFileInfo } from '@/modules/fileuploads/domain/operations' import { GetStreamBranchByName } from '@/modules/core/domain/branches/operations' import { EventBusEmit } from '@/modules/shared/services/eventBus' import { ModelEvents } from '@/modules/core/domain/branches/events' -import { fileUploadsLogger as logger } from '@/logging/logging' +import { fileUploadsLogger as logger } from '@/observability/logging' import { FileUploadConvertedStatus } from '@/modules/fileuploads/helpers/types' import { FileUploadInternalError } from '@/modules/fileuploads/helpers/errors' diff --git a/packages/server/modules/gatekeeper/index.ts b/packages/server/modules/gatekeeper/index.ts index fe482ea37..5cf8b3945 100644 --- a/packages/server/modules/gatekeeper/index.ts +++ b/packages/server/modules/gatekeeper/index.ts @@ -1,5 +1,5 @@ import cron from 'node-cron' -import { logger, moduleLogger } from '@/logging/logging' +import { logger, moduleLogger } from '@/observability/logging' import { SpeckleModule } from '@/modules/shared/helpers/typeHelper' import { getFeatureFlags } from '@/modules/shared/helpers/envHelper' import { validateModuleLicense } from '@/modules/gatekeeper/services/validateLicense' diff --git a/packages/server/modules/gatekeeper/services/subscriptions.ts b/packages/server/modules/gatekeeper/services/subscriptions.ts index 714be9f0e..6d5bfa9bf 100644 --- a/packages/server/modules/gatekeeper/services/subscriptions.ts +++ b/packages/server/modules/gatekeeper/services/subscriptions.ts @@ -1,4 +1,4 @@ -import { Logger } from '@/logging/logging' +import type { Logger } from '@/observability/logging' import { GetWorkspacePlan, GetWorkspacePlanPrice, diff --git a/packages/server/modules/gatekeeper/tests/unit/subscriptions.spec.ts b/packages/server/modules/gatekeeper/tests/unit/subscriptions.spec.ts index 1532e1de3..f947c9730 100644 --- a/packages/server/modules/gatekeeper/tests/unit/subscriptions.spec.ts +++ b/packages/server/modules/gatekeeper/tests/unit/subscriptions.spec.ts @@ -1,4 +1,4 @@ -import { logger } from '@/logging/logging' +import { testLogger as logger } from '@/observability/logging' import { SubscriptionData, SubscriptionDataInput, diff --git a/packages/server/modules/gendo/index.ts b/packages/server/modules/gendo/index.ts index 9380ded8f..40d90a289 100644 --- a/packages/server/modules/gendo/index.ts +++ b/packages/server/modules/gendo/index.ts @@ -1,5 +1,5 @@ import { SpeckleModule } from '@/modules/shared/helpers/typeHelper' -import { moduleLogger } from '@/logging/logging' +import { moduleLogger } from '@/observability/logging' import { getFeatureFlags } from '@/modules/shared/helpers/envHelper' import restApi from '@/modules/gendo/rest/index' diff --git a/packages/server/modules/index.ts b/packages/server/modules/index.ts index 411a588e1..1cc22d9aa 100644 --- a/packages/server/modules/index.ts +++ b/packages/server/modules/index.ts @@ -8,7 +8,7 @@ import { values, merge, camelCase, reduce, intersection } from 'lodash' import baseTypeDefs from '@/modules/core/graph/schema/baseTypeDefs' import { scalarResolvers } from '@/modules/core/graph/scalars' import { makeExecutableSchema } from '@graphql-tools/schema' -import { moduleLogger } from '@/logging/logging' +import { moduleLogger } from '@/observability/logging' import { addMocksToSchema } from '@graphql-tools/mock' import { getFeatureFlags } from '@/modules/shared/helpers/envHelper' import { isNonNullable } from '@speckle/shared' diff --git a/packages/server/modules/multiregion/index.ts b/packages/server/modules/multiregion/index.ts index 5eed2ead3..de4dbd1f3 100644 --- a/packages/server/modules/multiregion/index.ts +++ b/packages/server/modules/multiregion/index.ts @@ -1,4 +1,4 @@ -import { moduleLogger } from '@/logging/logging' +import { moduleLogger } from '@/observability/logging' import { getValidDefaultProjectRegionKey, initializeRegisteredRegionClients as initDb diff --git a/packages/server/modules/notifications/index.ts b/packages/server/modules/notifications/index.ts index 4d9c41e26..4968ec591 100644 --- a/packages/server/modules/notifications/index.ts +++ b/packages/server/modules/notifications/index.ts @@ -10,7 +10,7 @@ import { } from '@/modules/notifications/helpers/types' import { SpeckleModule } from '@/modules/shared/helpers/typeHelper' import { shouldDisableNotificationsConsumption } from '@/modules/shared/helpers/envHelper' -import { moduleLogger } from '@/logging/logging' +import { moduleLogger } from '@/observability/logging' export async function initializeConsumption( customHandlers?: Partial diff --git a/packages/server/modules/notifications/services/queue.ts b/packages/server/modules/notifications/services/queue.ts index 6ab010a69..b9691c019 100644 --- a/packages/server/modules/notifications/services/queue.ts +++ b/packages/server/modules/notifications/services/queue.ts @@ -16,7 +16,7 @@ import { isProdEnv, isTestEnv } from '@/modules/shared/helpers/envHelper' import Bull from 'bull' import { buildBaseQueueOptions } from '@/modules/shared/helpers/bullHelper' import cryptoRandomString from 'crypto-random-string' -import { logger, notificationsLogger, Observability } from '@/logging/logging' +import { logger, notificationsLogger, Observability } from '@/observability/logging' import { ensureErrorOrWrapAsCause } from '@/modules/shared/errors/ensureError' export type NotificationJobResult = { diff --git a/packages/server/modules/previews/index.ts b/packages/server/modules/previews/index.ts index 8b735c02b..b8cfa4ccc 100644 --- a/packages/server/modules/previews/index.ts +++ b/packages/server/modules/previews/index.ts @@ -2,7 +2,7 @@ import { validateScopes, authorizeResolver } from '@/modules/shared' import { makeOgImage } from '@/modules/previews/ogImage' -import { moduleLogger } from '@/logging/logging' +import { moduleLogger } from '@/observability/logging' import { messageProcessor } from '@/modules/previews/resultListener' import { db } from '@/db/knex' import { diff --git a/packages/server/modules/pwdreset/index.ts b/packages/server/modules/pwdreset/index.ts index 596ea08f5..337b6747a 100644 --- a/packages/server/modules/pwdreset/index.ts +++ b/packages/server/modules/pwdreset/index.ts @@ -1,4 +1,4 @@ -import { moduleLogger } from '@/logging/logging' +import { moduleLogger } from '@/observability/logging' import RestSetup from '@/modules/pwdreset/rest' import { SpeckleModule } from '@/modules/shared/helpers/typeHelper' import { noop } from 'lodash' diff --git a/packages/server/modules/serverinvites/index.ts b/packages/server/modules/serverinvites/index.ts index 73c70e001..c720a767c 100644 --- a/packages/server/modules/serverinvites/index.ts +++ b/packages/server/modules/serverinvites/index.ts @@ -1,5 +1,5 @@ import { registerOrUpdateScopeFactory } from '@/modules/shared/repositories/scopes' -import { moduleLogger } from '@/logging/logging' +import { moduleLogger } from '@/observability/logging' import db from '@/db/knex' import { Scopes } from '@speckle/shared' import { SpeckleModule } from '@/modules/shared/helpers/typeHelper' diff --git a/packages/server/modules/shared/middleware/index.ts b/packages/server/modules/shared/middleware/index.ts index 1d7b9e71a..c6ab8af24 100644 --- a/packages/server/modules/shared/middleware/index.ts +++ b/packages/server/modules/shared/middleware/index.ts @@ -33,7 +33,7 @@ import { Netmask } from 'netmask' import { Merge } from 'type-fest' import { resourceAccessRuleToIdentifier } from '@/modules/core/helpers/token' import { delayGraphqlResponsesBy } from '@/modules/shared/helpers/envHelper' -import { subscriptionLogger } from '@/logging/logging' +import { subscriptionLogger } from '@/observability/logging' import { GetUser } from '@/modules/core/domain/users/operations' import { validateTokenFactory } from '@/modules/core/services/tokens' import { diff --git a/packages/server/modules/shared/redis/redis.ts b/packages/server/modules/shared/redis/redis.ts index 285b455e4..2a8022309 100644 --- a/packages/server/modules/shared/redis/redis.ts +++ b/packages/server/modules/shared/redis/redis.ts @@ -1,4 +1,4 @@ -import { redisLogger } from '@/logging/logging' +import { redisLogger } from '@/observability/logging' import Redis, { RedisOptions } from 'ioredis' import { EnvironmentResourceError, diff --git a/packages/server/modules/shared/utils/mixpanel.ts b/packages/server/modules/shared/utils/mixpanel.ts index 4b722ea20..c7e066d09 100644 --- a/packages/server/modules/shared/utils/mixpanel.ts +++ b/packages/server/modules/shared/utils/mixpanel.ts @@ -8,7 +8,7 @@ import { import Mixpanel from 'mixpanel' import type express from 'express' import type http from 'http' -import { mixpanelLogger } from '@/logging/logging' +import { mixpanelLogger } from '@/observability/logging' let client: Optional = undefined let baseTrackingProperties: Optional> = undefined diff --git a/packages/server/modules/stats/index.ts b/packages/server/modules/stats/index.ts index c30676681..f659413cc 100644 --- a/packages/server/modules/stats/index.ts +++ b/packages/server/modules/stats/index.ts @@ -1,4 +1,4 @@ -import { moduleLogger } from '@/logging/logging' +import { moduleLogger } from '@/observability/logging' import { SpeckleModule } from '@/modules/shared/helpers/typeHelper' export const init: SpeckleModule['init'] = async () => { diff --git a/packages/server/modules/webhooks/index.ts b/packages/server/modules/webhooks/index.ts index 8cb439d62..bc019d215 100644 --- a/packages/server/modules/webhooks/index.ts +++ b/packages/server/modules/webhooks/index.ts @@ -1,6 +1,6 @@ import cron from 'node-cron' import { SpeckleModule } from '@/modules/shared/helpers/typeHelper' -import { activitiesLogger, moduleLogger } from '@/logging/logging' +import { activitiesLogger, moduleLogger } from '@/observability/logging' import { scheduleExecutionFactory } from '@/modules/core/services/taskScheduler' import { acquireTaskLockFactory, diff --git a/packages/server/modules/workspaces/events/eventListener.ts b/packages/server/modules/workspaces/events/eventListener.ts index 5b8914012..4c45930ce 100644 --- a/packages/server/modules/workspaces/events/eventListener.ts +++ b/packages/server/modules/workspaces/events/eventListener.ts @@ -21,7 +21,7 @@ import { isProjectResourceTarget, resolveTarget } from '@/modules/serverinvites/helpers/core' -import { logger, moduleLogger } from '@/logging/logging' +import { logger, moduleLogger } from '@/observability/logging' import { updateWorkspaceRoleFactory } from '@/modules/workspaces/services/management' import { EventPayload, getEventBus } from '@/modules/shared/services/eventBus' import { WorkspaceInviteResourceType } from '@/modules/workspacesCore/domain/constants' diff --git a/packages/server/modules/workspaces/index.ts b/packages/server/modules/workspaces/index.ts index fb838a643..cde177e03 100644 --- a/packages/server/modules/workspaces/index.ts +++ b/packages/server/modules/workspaces/index.ts @@ -1,4 +1,4 @@ -import { moduleLogger } from '@/logging/logging' +import { moduleLogger } from '@/observability/logging' import { getFeatureFlags } from '@/modules/shared/helpers/envHelper' import { registerOrUpdateScopeFactory } from '@/modules/shared/repositories/scopes' import db from '@/db/knex' diff --git a/packages/server/modules/workspaces/services/sso.ts b/packages/server/modules/workspaces/services/sso.ts index 3bffcb78f..1d877ff8e 100644 --- a/packages/server/modules/workspaces/services/sso.ts +++ b/packages/server/modules/workspaces/services/sso.ts @@ -38,7 +38,7 @@ import { getEmailFromOidcProfile, isValidSsoSession } from '@/modules/workspaces/domain/sso/logic' -import { Logger } from '@/logging/logging' +import type { Logger } from '@/observability/logging' // this probably should go a lean validation endpoint too const validateOidcProviderAttributes = ({ diff --git a/packages/server/modules/workspacesCore/index.ts b/packages/server/modules/workspacesCore/index.ts index 2b9aa37f3..e1ddc7ded 100644 --- a/packages/server/modules/workspacesCore/index.ts +++ b/packages/server/modules/workspacesCore/index.ts @@ -1,5 +1,5 @@ import { SpeckleModule } from '@/modules/shared/helpers/typeHelper' -import { moduleLogger } from '@/logging/logging' +import { moduleLogger } from '@/observability/logging' export const init: SpeckleModule['init'] = () => { moduleLogger.info('⚒️ Init workspaces core module') diff --git a/packages/server/logging/apolloSubscriptions.ts b/packages/server/observability/components/apollo/apolloSubscriptions.ts similarity index 92% rename from packages/server/logging/apolloSubscriptions.ts rename to packages/server/observability/components/apollo/apolloSubscriptions.ts index f95e5bdb4..c06724673 100644 --- a/packages/server/logging/apolloSubscriptions.ts +++ b/packages/server/observability/components/apollo/apolloSubscriptions.ts @@ -1,13 +1,16 @@ /* eslint-disable camelcase */ import type { GraphQLContext } from '@/modules/shared/helpers/typeHelper' import type { ExecutionParams } from 'subscriptions-transport-ws' -import { shouldLogAsInfoLevel, shouldLogAsWarnLevel } from '@/logging/graphqlError' +import { + shouldLogAsInfoLevel, + shouldLogAsWarnLevel +} from '@/observability/utils/logLevels' import { BaseError } from '@/modules/shared/errors' import { GraphQLError } from 'graphql' -import { redactSensitiveVariables } from '@/logging/loggingHelper' +import { redactSensitiveVariables } from '@/observability/utils/redact' import type { Counter } from 'prom-client' -import { getRequestContext } from '@/logging/requestContext' -import { subscriptionLogger } from '@/logging/logging' +import { getRequestContext } from '@/observability/components/express/requestContext' +import { subscriptionLogger } from '@/observability/logging' // eslint-disable-next-line @typescript-eslint/no-explicit-any type SubscriptionResponse = { errors?: GraphQLError[]; data?: any } diff --git a/packages/server/logging/apolloSubscriptionMonitoring.ts b/packages/server/observability/components/apollo/metrics/apolloSubscriptionMonitoring.ts similarity index 100% rename from packages/server/logging/apolloSubscriptionMonitoring.ts rename to packages/server/observability/components/apollo/metrics/apolloSubscriptionMonitoring.ts diff --git a/packages/server/logging/expressLogging.ts b/packages/server/observability/components/express/expressLogging.ts similarity index 99% rename from packages/server/logging/expressLogging.ts rename to packages/server/observability/components/express/expressLogging.ts index e5a7eaf48..a376ed0c6 100644 --- a/packages/server/logging/expressLogging.ts +++ b/packages/server/observability/components/express/expressLogging.ts @@ -1,4 +1,4 @@ -import { logger } from '@/logging/logging' +import { logger } from '@/observability/logging' import { randomUUID } from 'crypto' import HttpLogger from 'pino-http' import type { NextFunction, Response } from 'express' diff --git a/packages/server/logging/errorMetrics.ts b/packages/server/observability/components/express/metrics/errorMetrics.ts similarity index 100% rename from packages/server/logging/errorMetrics.ts rename to packages/server/observability/components/express/metrics/errorMetrics.ts diff --git a/packages/server/logging/requestContext.ts b/packages/server/observability/components/express/requestContext.ts similarity index 93% rename from packages/server/logging/requestContext.ts rename to packages/server/observability/components/express/requestContext.ts index b383d08c4..93b175445 100644 --- a/packages/server/logging/requestContext.ts +++ b/packages/server/observability/components/express/requestContext.ts @@ -1,4 +1,4 @@ -import { REQUEST_ID_HEADER } from '@/logging/expressLogging' +import { REQUEST_ID_HEADER } from '@/observability/components/express/expressLogging' import { asyncRequestContextEnabled } from '@/modules/shared/helpers/envHelper' import type express from 'express' import { AsyncLocalStorage } from 'node:async_hooks' diff --git a/packages/server/logging/highFrequencyMetrics/heapSizeAndUsed.ts b/packages/server/observability/components/highFrequencyMetrics/heapSizeAndUsed.ts similarity index 97% rename from packages/server/logging/highFrequencyMetrics/heapSizeAndUsed.ts rename to packages/server/observability/components/highFrequencyMetrics/heapSizeAndUsed.ts index e01e9e892..f820faef5 100644 --- a/packages/server/logging/highFrequencyMetrics/heapSizeAndUsed.ts +++ b/packages/server/observability/components/highFrequencyMetrics/heapSizeAndUsed.ts @@ -17,7 +17,7 @@ */ import { Histogram, Registry } from 'prom-client' -import type { Metric } from '@/logging/highFrequencyMetrics/highfrequencyMonitoring' +import type { Metric } from '@/observability/components/highFrequencyMetrics/highfrequencyMonitoring' const NODEJS_HEAP_SIZE_TOTAL = 'nodejs_heap_size_total_bytes_high_frequency' const NODEJS_HEAP_SIZE_USED = 'nodejs_heap_size_used_bytes_high_frequency' diff --git a/packages/server/logging/highFrequencyMetrics/highfrequencyMonitoring.ts b/packages/server/observability/components/highFrequencyMetrics/highfrequencyMonitoring.ts similarity index 87% rename from packages/server/logging/highFrequencyMetrics/highfrequencyMonitoring.ts rename to packages/server/observability/components/highFrequencyMetrics/highfrequencyMonitoring.ts index b11bfe6cd..8ede14d8e 100644 --- a/packages/server/logging/highFrequencyMetrics/highfrequencyMonitoring.ts +++ b/packages/server/observability/components/highFrequencyMetrics/highfrequencyMonitoring.ts @@ -5,9 +5,9 @@ */ import { Histogram, Registry } from 'prom-client' -import { processCpuTotal } from '@/logging/highFrequencyMetrics/processCPUTotal' -import { heapSizeAndUsed } from '@/logging/highFrequencyMetrics/heapSizeAndUsed' -import { knexConnections } from '@/logging/highFrequencyMetrics/knexConnectionPool' +import { processCpuTotal } from '@/observability/components/highFrequencyMetrics/processCPUTotal' +import { heapSizeAndUsed } from '@/observability/components/highFrequencyMetrics/heapSizeAndUsed' +import { knexConnections } from '@/observability/components/highFrequencyMetrics/knexConnectionPool' import { type Knex } from 'knex' type MetricConfig = { diff --git a/packages/server/logging/highFrequencyMetrics/knexConnectionPool.ts b/packages/server/observability/components/highFrequencyMetrics/knexConnectionPool.ts similarity index 97% rename from packages/server/logging/highFrequencyMetrics/knexConnectionPool.ts rename to packages/server/observability/components/highFrequencyMetrics/knexConnectionPool.ts index 6a6afc5fd..3130029b3 100644 --- a/packages/server/logging/highFrequencyMetrics/knexConnectionPool.ts +++ b/packages/server/observability/components/highFrequencyMetrics/knexConnectionPool.ts @@ -1,5 +1,5 @@ import { Histogram, Registry } from 'prom-client' -import type { Metric } from '@/logging/highFrequencyMetrics/highfrequencyMonitoring' +import type { Metric } from '@/observability/components/highFrequencyMetrics/highfrequencyMonitoring' import type { Knex } from 'knex' import { numberOfFreeConnections } from '@/modules/shared/helpers/dbHelper' diff --git a/packages/server/logging/highFrequencyMetrics/processCPUTotal.ts b/packages/server/observability/components/highFrequencyMetrics/processCPUTotal.ts similarity index 97% rename from packages/server/logging/highFrequencyMetrics/processCPUTotal.ts rename to packages/server/observability/components/highFrequencyMetrics/processCPUTotal.ts index 6d1f44144..b5dacbd7a 100644 --- a/packages/server/logging/highFrequencyMetrics/processCPUTotal.ts +++ b/packages/server/observability/components/highFrequencyMetrics/processCPUTotal.ts @@ -17,7 +17,7 @@ */ import { Histogram, Registry } from 'prom-client' -import type { Metric } from '@/logging/highFrequencyMetrics/highfrequencyMonitoring' +import type { Metric } from '@/observability/components/highFrequencyMetrics/highfrequencyMonitoring' const PROCESS_CPU_USER_SECONDS = 'process_cpu_user_seconds_total_high_frequency' const PROCESS_CPU_SYSTEM_SECONDS = 'process_cpu_system_seconds_total_high_frequency' diff --git a/packages/server/logging/httpServerMonitoring.ts b/packages/server/observability/components/httpServer/httpServerMonitoring.ts similarity index 100% rename from packages/server/logging/httpServerMonitoring.ts rename to packages/server/observability/components/httpServer/httpServerMonitoring.ts diff --git a/packages/server/logging/knexMonitoring.ts b/packages/server/observability/components/knex/knexMonitoring.ts similarity index 99% rename from packages/server/logging/knexMonitoring.ts rename to packages/server/observability/components/knex/knexMonitoring.ts index 604eb6945..044cc70ff 100644 --- a/packages/server/logging/knexMonitoring.ts +++ b/packages/server/observability/components/knex/knexMonitoring.ts @@ -4,7 +4,7 @@ import { type Knex } from 'knex' import { Logger } from 'pino' import { toNDecimalPlaces } from '@/modules/core/utils/formatting' import { omit } from 'lodash' -import { getRequestContext } from '@/logging/requestContext' +import { getRequestContext } from '@/observability/components/express/requestContext' import { collectLongTrace } from '@speckle/shared' let metricQueryDuration: prometheusClient.Summary diff --git a/packages/server/logging/index.ts b/packages/server/observability/index.ts similarity index 86% rename from packages/server/logging/index.ts rename to packages/server/observability/index.ts index 148bb54ef..e38237b22 100644 --- a/packages/server/logging/index.ts +++ b/packages/server/observability/index.ts @@ -2,10 +2,10 @@ import prometheusClient from 'prom-client' import promBundle from 'express-prom-bundle' -import { initKnexPrometheusMetrics } from '@/logging/knexMonitoring' -import { initHighFrequencyMonitoring } from '@/logging/highFrequencyMetrics/highfrequencyMonitoring' +import { initKnexPrometheusMetrics } from '@/observability/components/knex/knexMonitoring' +import { initHighFrequencyMonitoring } from '@/observability/components/highFrequencyMetrics/highfrequencyMonitoring' import { highFrequencyMetricsCollectionPeriodMs } from '@/modules/shared/helpers/envHelper' -import { startupLogger as logger } from '@/logging/logging' +import { startupLogger as logger } from '@/observability/logging' import type express from 'express' import { getAllRegisteredDbClients } from '@/modules/multiregion/utils/dbSelector' diff --git a/packages/server/logging/logging.ts b/packages/server/observability/logging.ts similarity index 93% rename from packages/server/logging/logging.ts rename to packages/server/observability/logging.ts index 1ab0b98fa..8e0af6f63 100644 --- a/packages/server/logging/logging.ts +++ b/packages/server/observability/logging.ts @@ -33,6 +33,8 @@ export const subscriptionLogger = extendLoggerComponent(logger, 'subscription') export const healthCheckLogger = extendLoggerComponent(logger, 'healthcheck') export const testLogger = extendLoggerComponent(logger, 'test') export const fileUploadsLogger = extendLoggerComponent(logger, 'file-uploads') +export const emailLogger = extendLoggerComponent(logger, 'email') +export const taskSchedulerLogger = extendLoggerComponent(logger, 'task-scheduler') export type Logger = typeof logger export { extendLoggerComponent, Observability } diff --git a/packages/server/otel.ts b/packages/server/observability/otel.ts similarity index 100% rename from packages/server/otel.ts rename to packages/server/observability/otel.ts diff --git a/packages/server/logging/graphqlError.ts b/packages/server/observability/utils/logLevels.ts similarity index 100% rename from packages/server/logging/graphqlError.ts rename to packages/server/observability/utils/logLevels.ts diff --git a/packages/server/logging/machineId.ts b/packages/server/observability/utils/machineId.ts similarity index 100% rename from packages/server/logging/machineId.ts rename to packages/server/observability/utils/machineId.ts diff --git a/packages/server/logging/loggingHelper.spec.ts b/packages/server/observability/utils/redact.spec.ts similarity index 95% rename from packages/server/logging/loggingHelper.spec.ts rename to packages/server/observability/utils/redact.spec.ts index c05ffc5de..510b9c748 100644 --- a/packages/server/logging/loggingHelper.spec.ts +++ b/packages/server/observability/utils/redact.spec.ts @@ -1,7 +1,7 @@ import { expect } from 'chai' -import { redactSensitiveVariables } from '@/logging/loggingHelper' +import { redactSensitiveVariables } from '@/observability/utils/redact' -describe('loggingHelper', () => { +describe('redact', () => { describe('filterSensitiveVariables', () => { it('should filter sensitive variables at root', () => { const variables = { diff --git a/packages/server/logging/loggingHelper.ts b/packages/server/observability/utils/redact.ts similarity index 100% rename from packages/server/logging/loggingHelper.ts rename to packages/server/observability/utils/redact.ts diff --git a/packages/server/scripts/duplicateUserMigration.js b/packages/server/scripts/duplicateUserMigration.js index b6d8da7f0..eb6f10104 100644 --- a/packages/server/scripts/duplicateUserMigration.js +++ b/packages/server/scripts/duplicateUserMigration.js @@ -1,5 +1,5 @@ const { knex } = require('@/db/knex') -const { logger } = require('@/logging/logging') +const { logger } = require('@/observability/logging') const roles = require('@/modules/core/roles.js') const { Roles } = require('@speckle/shared') diff --git a/packages/server/scripts/seedUsers.js b/packages/server/scripts/seedUsers.js index 610c8b439..b7149f9ac 100644 --- a/packages/server/scripts/seedUsers.js +++ b/packages/server/scripts/seedUsers.js @@ -1,6 +1,6 @@ require('../bootstrap') const { db } = require('@/db/knex') -const { logger } = require('@/logging/logging') +const { logger } = require('@/observability/logging') const { getServerInfoFactory } = require('@/modules/core/repositories/server') const { findEmailFactory, diff --git a/packages/server/scripts/streamObjects.js b/packages/server/scripts/streamObjects.js index 544ed287e..dd6f48b8e 100644 --- a/packages/server/scripts/streamObjects.js +++ b/packages/server/scripts/streamObjects.js @@ -4,7 +4,7 @@ const { fetch } = require('undici') const { init } = require(`@/app`) const request = require('supertest') const { exit } = require('yargs') -const { logger } = require('@/logging/logging') +const { logger } = require('@/observability/logging') const { Scopes } = require('@speckle/shared') const { getStreamFactory, diff --git a/packages/server/test/helpers.js b/packages/server/test/helpers.js index 120233847..f89f4c536 100644 --- a/packages/server/test/helpers.js +++ b/packages/server/test/helpers.js @@ -1,5 +1,5 @@ /* istanbul ignore file */ -// const { logger } = require('@/logging/logging') +// const { logger } = require('@/observability/logging') const crypto = require('crypto') function generateManyObjects(shitTon, noise) { diff --git a/packages/server/test/hooks.ts b/packages/server/test/hooks.ts index a632c86d4..bfa1324c4 100644 --- a/packages/server/test/hooks.ts +++ b/packages/server/test/hooks.ts @@ -12,7 +12,7 @@ import deepEqualInAnyOrder from 'deep-equal-in-any-order' import { knex as mainDb } from '@/db/knex' import { init, startHttp, shutdown } from '@/app' import graphqlChaiPlugin from '@/test/plugins/graphql' -import { logger } from '@/logging/logging' +import { testLogger as logger } from '@/observability/logging' import { once } from 'events' import type http from 'http' import type express from 'express' diff --git a/packages/server/test/notificationsHelper.ts b/packages/server/test/notificationsHelper.ts index fb2d7d651..559ec7979 100644 --- a/packages/server/test/notificationsHelper.ts +++ b/packages/server/test/notificationsHelper.ts @@ -1,4 +1,4 @@ -import { logger } from '@/logging/logging' +import { notificationsLogger as logger } from '@/observability/logging' import { getQueue, NotificationJobResult } from '@/modules/notifications/services/queue' import { EventEmitter } from 'events' import { CompletedEventCallback, FailedEventCallback, JobId } from 'bull' diff --git a/packages/server/tsconfig.json b/packages/server/tsconfig.json index 8605ec88e..71b6b0644 100644 --- a/packages/server/tsconfig.json +++ b/packages/server/tsconfig.json @@ -105,18 +105,15 @@ "swc": true }, "include": [ + "bin/**/*", "db/**/*", "healthchecks/**/*", - "logging/**/*", "modules/**/*", - "bin/**/*", + "observability/**/*", "scripts/**/*", - "type-augmentations/**/*", "test/**/*", + "type-augmentations/**/*", "app.ts", - "otel.ts", - "bootstrap.ts", - "knexfile.ts", "bootstrap.js", "knexfile.ts" ], diff --git a/packages/shared/src/observability/index.ts b/packages/shared/src/observability/index.ts index 23aaf5e13..dbac64818 100644 --- a/packages/shared/src/observability/index.ts +++ b/packages/shared/src/observability/index.ts @@ -1,12 +1,25 @@ import { pino } from 'pino' import type { LoggerOptions } from 'pino' -import { toClef, clefLevels } from './pinoClef.js' +import { toClef, toClefLogLevel } from './pinoClef.js' let logger: pino.Logger -type MixinFn = (mergeObject: object, level: number) => object +export type MixinFn = (mergeObject: object, level: number) => object +type LogLevelFormatter = (label: string, number: number) => object +type LogFormatter = (logObject: Record) => Record + +const defaultLevelFormatterFactory = + (pretty: boolean): LogLevelFormatter => + (label, number) => + // for not pretty, we're providing clef levels + pretty ? { level: label } : toClefLogLevel(number) + +const defaultLogFormatterFactory = + (pretty: boolean): LogFormatter => + (logObject) => + pretty ? logObject : toClef(logObject) export function getLogger( - logLevel = 'info', + minimumLoggedLevel = 'info', pretty = false, mixin?: MixinFn ): pino.Logger { @@ -15,22 +28,13 @@ export function getLogger( const pinoOptions: LoggerOptions = { base: undefined, // Set to undefined to avoid adding pid, hostname properties to each log. formatters: { - level: (label: string, number: number) => - // for not pretty, we're providing clef levels - pretty - ? { level: label } - : { - '@l': - number in clefLevels - ? clefLevels[number as keyof typeof clefLevels] - : clefLevels[30] - }, - log: (logObject) => (pretty ? logObject : toClef(logObject)) + level: defaultLevelFormatterFactory(pretty), + log: defaultLogFormatterFactory(pretty) }, mixin, // when not pretty, to produce a clef format, we need the message to be the message template key messageKey: pretty ? 'msg' : '@mt', - level: logLevel, + level: minimumLoggedLevel, // when not pretty, we need the time in the clef appropriate field, not from pino timestamp: pretty ? pino.stdTimeFunctions.isoTime : false } diff --git a/packages/shared/src/observability/pinoClef.ts b/packages/shared/src/observability/pinoClef.ts index 55d43f4aa..c2f3d6cfb 100644 --- a/packages/shared/src/observability/pinoClef.ts +++ b/packages/shared/src/observability/pinoClef.ts @@ -1,4 +1,4 @@ -export const clefLevels = { +const clefLevels = { 60: 'Fatal', // FATAL 50: 'Error', // ERROR 40: 'Warning', // WARN @@ -7,6 +7,15 @@ export const clefLevels = { 10: 'Verbose' // TRACE / Verbose } as const +export function toClefLogLevel(number: number) { + return { + '@l': + number in clefLevels + ? clefLevels[number as keyof typeof clefLevels] + : clefLevels[30] + } +} + export function toClef(log: Record): Record { // add the stuff CLEF wants .... if (log.time) {