refactor(shared): unified queue handling (#4691)

* feat(shared): unified queue initialization in shared

* feat(queues): use the new queue creation everywhere

* chore(shared): move to redis module

* chore(shared): fix export maps

* chore(fileimport): add deps properly

* fix(shared): import fix

* fix(everything): moear imports

* fix(server): cjs imports

packages/shared/src/queue/config.ts

@@ -0,0 +1,69 @@
+import Bull from 'bull'
+import { Redis } from 'ioredis'
+import { isRedisReady } from '../redis/isRedisReady.js'
+
+// we're caching this here, so that there is one client for the app lifecycle
+
+type ClientCache = Record<string, { client?: Redis; subscriber?: Redis }>
+
+const clientCache: ClientCache = {}
+
+export const initializeQueue = async <T>({
+  queueName,
+  redisUrl,
+  options
+}: {
+  queueName: string
+  redisUrl: string
+  options?: Partial<Bull.QueueOptions>
+}): Promise<Bull.Queue<T>> => {
+  if (!(redisUrl in clientCache)) clientCache[redisUrl] = {}
+  const opts: Bull.QueueOptions = {
+    ...options,
+    // redisOpts here will contain at least a property of connectionName which will identify the queue based on its name
+    createClient(type, redisOpts) {
+      switch (type) {
+        case 'client':
+          if (redisUrl in clientCache && clientCache[redisUrl].client !== undefined) {
+            return clientCache[redisUrl].client
+          } else {
+            const client = new Redis(redisUrl, redisOpts ?? {})
+            clientCache[redisUrl].client = client
+            return client
+          }
+        case 'subscriber':
+          if (
+            redisUrl in clientCache &&
+            clientCache[redisUrl].subscriber !== undefined
+          ) {
+            return clientCache[redisUrl].subscriber
+          } else {
+            const subscriber = new Redis(redisUrl, {
+              ...redisOpts,
+              maxRetriesPerRequest: null,
+              enableReadyCheck: false
+            })
+            clientCache[redisUrl].subscriber = subscriber
+            return subscriber
+          }
+        case 'bclient':
+          return new Redis(redisUrl, {
+            ...redisOpts,
+            maxRetriesPerRequest: null,
+            enableReadyCheck: false
+          })
+        default:
+          throw new Error(`Unexpected connection type: ${type}`)
+      }
+    }
+  }
+  const newQueue = new Bull<T>(queueName, opts)
+  // bull does not check if redis is ready...
+  //
+  // logger.info('Checking Redis connection is ready...')
+  if (!clientCache[redisUrl].client)
+    throw new Error('Redis client not properly initialized')
+  await isRedisReady(clientCache[redisUrl].client)
+  // await isRedisReady(clientCache[redisUrl].subscriber)
+  return await newQueue.isReady()
+}

packages/shared/src/queue/index.ts

@@ -0,0 +1 @@
+export * from './config.js'

packages/shared/src/redis/index.ts

@@ -0,0 +1 @@
+export * from './isRedisReady.js'

packages/shared/src/redis/isRedisReady.ts

@@ -0,0 +1,35 @@
+import { Redis } from 'ioredis'
+import { ensureError } from '../core/helpers/error.js'
+
+// MIT Licensed: https://github.com/OptimalBits/bull/blob/develop/LICENSE.md
+// Reference: https://github.com/OptimalBits/bull/blob/develop/lib/utils.js
+export const isRedisReady = (client: Redis) => {
+  return new Promise<void>((resolve, reject) => {
+    if (client.status === 'ready') {
+      resolve()
+    } else {
+      function handleReady() {
+        client.removeListener('end', handleEnd)
+        client.removeListener('error', handleError)
+        resolve()
+      }
+
+      function handleError(e: unknown) {
+        const err = ensureError(e, 'Unknown error in Redis client')
+        client.removeListener('ready', handleReady)
+        client.removeListener('error', handleError)
+        reject(err)
+      }
+
+      function handleEnd() {
+        client.removeListener('ready', handleReady)
+        client.removeListener('error', handleError)
+        reject(new Error('Redis connection ended'))
+      }
+
+      client.once('ready', handleReady)
+      client.on('error', handleError)
+      client.once('end', handleEnd)
+    }
+  })
+}

packages/shared/src/workers/fileimport/index.ts

@@ -0,0 +1 @@
+export * from './job.js'

packages/shared/src/workers/index.ts

@@ -0,0 +1 @@
+export * from './state.js'

packages/shared/src/workers/previews/index.ts

@@ -0,0 +1,2 @@
+export * from './interface.js'
+export * from './job.js'

packages/shared/src/workers/state.ts

@@ -0,0 +1,6 @@
+export const AppState = {
+  STARTING: 'starting',
+  RUNNING: 'running',
+  SHUTTINGDOWN: 'shuttingdown'
+} as const
+export type AppState = (typeof AppState)[keyof typeof AppState]