From bff8aef606423917071ac178cd6d09591fd42059 Mon Sep 17 00:00:00 2001 From: AlexandruPopovici Date: Mon, 3 Oct 2022 11:29:55 +0300 Subject: [PATCH 01/54] Started working on progressive sao --- packages/viewer-sandbox/src/main.ts | 4 +- .../viewer/src/modules/SpeckleRenderer.ts | 49 ++++++++++++++++++- .../src/modules/pipeline/ApplySAOPass.ts | 20 ++++---- .../viewer/src/modules/pipeline/Pipeline.ts | 15 ++++-- ...kleSAOPass.ts => SpeckleDynamicSAOPass.ts} | 8 +-- 5 files changed, 76 insertions(+), 20 deletions(-) rename packages/viewer/src/modules/pipeline/{SpeckleSAOPass.ts => SpeckleDynamicSAOPass.ts} (98%) diff --git a/packages/viewer-sandbox/src/main.ts b/packages/viewer-sandbox/src/main.ts index 2384b49f2..067770572 100644 --- a/packages/viewer-sandbox/src/main.ts +++ b/packages/viewer-sandbox/src/main.ts @@ -82,14 +82,14 @@ sandbox.makeFilteringUI() await sandbox.loadUrl( // 'https://speckle.xyz/streams/da9e320dad/commits/5388ef24b8?c=%5B-7.66134,10.82932,6.41935,-0.07739,-13.88552,1.8697,0,1%5D' // Revit sample house (good for bim-like stuff with many display meshes) - // 'https://speckle.xyz/streams/da9e320dad/commits/5388ef24b8' + 'https://speckle.xyz/streams/da9e320dad/commits/5388ef24b8' // 'Super' heavy revit shit // 'https://speckle.xyz/streams/e6f9156405/commits/0694d53bb5' // Same sample revit house, local to dim's computer // 'http://localhost:3000/streams/6960695d7b/commits/da0a2343fa' // 'http://100.66.180.109:3000/streams/6960695d7b/commits/417526751d' // IFC building (good for a tree based structure) - 'https://latest.speckle.dev/streams/92b620fb17/commits/2ebd336223' + // 'https://latest.speckle.dev/streams/92b620fb17/commits/2ebd336223' // IFC story, a subtree of the above // 'https://latest.speckle.dev/streams/92b620fb17/objects/8247bbc53865b0e0cb5ee4e252e66216' // Small scale lines diff --git a/packages/viewer/src/modules/SpeckleRenderer.ts b/packages/viewer/src/modules/SpeckleRenderer.ts index 2474ab464..ebbd91172 100644 --- a/packages/viewer/src/modules/SpeckleRenderer.ts +++ b/packages/viewer/src/modules/SpeckleRenderer.ts @@ -62,6 +62,13 @@ export default class SpeckleRenderer { private filterBatchRecording: string[] private pipeline: Pipeline + private lastAzimuth: number + private lastPolar: number + private lastDistance: number + private lastTarget: Vector3 = new Vector3() + private lasMaxCameraMotion: number + private readonly CAMERA_MOTION_EPSILON: number = 0.0001 + public get renderer(): WebGLRenderer { return this._renderer } @@ -170,10 +177,21 @@ export default class SpeckleRenderer { camHelper.name = 'CamHelper' helpers.add(camHelper) } + this.viewer.cameraHandler.controls.restThreshold = 0.001 + this.viewer.cameraHandler.controls.addEventListener('sleep', (event) => { + this.pipeline.onStationaryBegin() + }) + this.viewer.cameraHandler.controls.addEventListener('rest', (event) => { + this.pipeline.onStationaryBegin() + }) + this.viewer.cameraHandler.controls.addEventListener('wake', (event) => { + this.pipeline.onStationaryEnd() + }) } public update(deltaTime: number) { this.batcher.update(deltaTime) + const viewer = new Vector3() const viewerLow = new Vector3() const viewerHigh = new Vector3() @@ -270,7 +288,36 @@ export default class SpeckleRenderer { this.viewer.cameraHandler.activeCam.camera.updateProjectionMatrix() this.viewer.cameraHandler.camera.updateProjectionMatrix() this.pipeline.pipelineOptions = { saoParams: { saoScale: d } } - // console.log(d) + + // const currentAzimuth = this.viewer.cameraHandler.controls.azimuthAngle + // const currentPolar = this.viewer.cameraHandler.controls.polarAngle + // const currentDistance = this.viewer.cameraHandler.controls.distance + // const currentTarget = this.viewer.cameraHandler.controls.getTarget(new Vector3()) + // /** This looks so much better */ + // const dAzimuth = Math.max( + // 0, + // Math.abs(currentAzimuth - this.lastAzimuth) - this.CAMERA_MOTION_EPSILON + // ) + // const dPolar = Math.max( + // 0, + // Math.abs(currentPolar - this.lastPolar) - this.CAMERA_MOTION_EPSILON + // ) + // const dDistance = Math.max( + // 0, + // Math.abs(currentDistance - this.lastDistance) - this.CAMERA_MOTION_EPSILON + // ) + // const dTarget = currentTarget.distanceTo(this.lastTarget) + + // const motionMaxDelta = Math.max(0, Math.max(dAzimuth, dPolar, dDistance, dTarget)) + // if (motionMaxDelta === 0 && this.lasMaxCameraMotion > 0) { + // console.log('Stopped') + // } + // this.lasMaxCameraMotion = motionMaxDelta + + // this.lastAzimuth = currentAzimuth + // this.lastPolar = currentPolar + // this.lastDistance = currentDistance + // this.lastTarget.copy(currentTarget) } public render(camera: Camera) { diff --git a/packages/viewer/src/modules/pipeline/ApplySAOPass.ts b/packages/viewer/src/modules/pipeline/ApplySAOPass.ts index cbddfdc58..92c2ad7d7 100644 --- a/packages/viewer/src/modules/pipeline/ApplySAOPass.ts +++ b/packages/viewer/src/modules/pipeline/ApplySAOPass.ts @@ -24,16 +24,16 @@ export class ApplySAOPass extends Pass { fragmentShader: CopyShader.fragmentShader, blending: NoBlending }) - this.materialCopy.transparent = true - this.materialCopy.depthTest = false - this.materialCopy.depthWrite = false - this.materialCopy.blending = CustomBlending - this.materialCopy.blendSrc = DstColorFactor - this.materialCopy.blendDst = ZeroFactor - this.materialCopy.blendEquation = AddEquation - this.materialCopy.blendSrcAlpha = DstAlphaFactor - this.materialCopy.blendDstAlpha = ZeroFactor - this.materialCopy.blendEquationAlpha = AddEquation + // this.materialCopy.transparent = true + // this.materialCopy.depthTest = false + // this.materialCopy.depthWrite = false + // this.materialCopy.blending = CustomBlending + // this.materialCopy.blendSrc = DstColorFactor + // this.materialCopy.blendDst = ZeroFactor + // this.materialCopy.blendEquation = AddEquation + // this.materialCopy.blendSrcAlpha = DstAlphaFactor + // this.materialCopy.blendDstAlpha = ZeroFactor + // this.materialCopy.blendEquationAlpha = AddEquation this.materialCopy.uniforms['tDiffuse'].value = srcSao this.materialCopy.needsUpdate = true this.fsQuad = new FullScreenQuad(this.materialCopy) diff --git a/packages/viewer/src/modules/pipeline/Pipeline.ts b/packages/viewer/src/modules/pipeline/Pipeline.ts index e6991a1b4..02af84781 100644 --- a/packages/viewer/src/modules/pipeline/Pipeline.ts +++ b/packages/viewer/src/modules/pipeline/Pipeline.ts @@ -4,7 +4,7 @@ import { RenderPass } from 'three/examples/jsm/postprocessing/RenderPass.js' import { SAOPassParams } from 'three/examples/jsm/postprocessing/SAOPass.js' import Batcher from '../batching/Batcher' import { ApplySAOPass } from './ApplySAOPass' -import { NormalsType, SpeckleSAOPass } from './SpeckleSAOPass' +import { NormalsType, SpeckleDynamicSAOPass } from './SpeckleDynamicSAOPass' export interface PipelineOptions { saoEnabled?: boolean @@ -36,7 +36,7 @@ export class Pipeline { private _pipelineOptions: PipelineOptions = {} private composer: EffectComposer = null private renderPass: RenderPass = null - private saoPass: SpeckleSAOPass = null + private saoPass: SpeckleDynamicSAOPass = null private applySaoPass: ApplySAOPass = null private drawingSize: Vector2 = new Vector2() @@ -59,7 +59,7 @@ export class Pipeline { } public configure(scene: Scene, camera: Camera) { - this.saoPass = new SpeckleSAOPass( + this.saoPass = new SpeckleDynamicSAOPass( scene, camera, this._batcher, @@ -69,6 +69,7 @@ export class Pipeline { this.composer.addPass(this.saoPass) this.renderPass = new RenderPass(scene, camera) this.renderPass.renderToScreen = true + this.renderPass.enabled = false this.composer.addPass(this.renderPass) this.applySaoPass = new ApplySAOPass(this.saoPass.saoRenderTarget.texture) this.applySaoPass.renderToScreen = true @@ -94,4 +95,12 @@ export class Pipeline { public resize(width: number, height: number) { this.composer.setSize(width, height) } + + public onStationaryBegin() { + this.saoPass.accumulate = true + } + + public onStationaryEnd() { + this.saoPass.accumulate = false + } } diff --git a/packages/viewer/src/modules/pipeline/SpeckleSAOPass.ts b/packages/viewer/src/modules/pipeline/SpeckleDynamicSAOPass.ts similarity index 98% rename from packages/viewer/src/modules/pipeline/SpeckleSAOPass.ts rename to packages/viewer/src/modules/pipeline/SpeckleDynamicSAOPass.ts index 4158efec9..e4a8bda2e 100644 --- a/packages/viewer/src/modules/pipeline/SpeckleSAOPass.ts +++ b/packages/viewer/src/modules/pipeline/SpeckleDynamicSAOPass.ts @@ -30,12 +30,13 @@ export enum NormalsType { * SAO implementation inspired from bhouston previous SAO work */ -export class SpeckleSAOPass extends SAOPass { +export class SpeckleDynamicSAOPass extends SAOPass { private _oldClearColor private prevStdDev private prevNumSamples private batcher: Batcher = null private normalsType: NormalsType = NormalsType.IMPROVED + public accumulate = false public set normalsRendering(type: NormalsType) { this.normalsType = type @@ -138,7 +139,7 @@ export class SpeckleSAOPass extends SAOPass { ) this.saoMaterial.uniforms['cameraProjectionMatrix'].value = this.camera.projectionMatrix - // this.saoMaterial.uniforms['randomSeed'].value = Math.random(); + // this.saoMaterial.uniforms['randomSeed'].value = Math.random() const depthCutoff = this.params.saoBlurDepthCutoff * @@ -242,12 +243,11 @@ export class SpeckleSAOPass extends SAOPass { // setup pass state renderer.autoClear = false - if (clearColor !== undefined && clearColor !== null) { + if (clearColor !== undefined && clearColor !== null && !this.accumulate) { renderer.setClearColor(clearColor) renderer.setClearAlpha(clearAlpha || 0.0) renderer.clear() } - ;(this.fsQuad as FullScreenQuad).material = passMaterial ;(this.fsQuad as FullScreenQuad).render(renderer) From b2610095d28ca7653d935d4b0db8a7498957748c Mon Sep 17 00:00:00 2001 From: AlexandruPopovici Date: Wed, 5 Oct 2022 16:52:52 +0300 Subject: [PATCH 02/54] Started working on the ao generation stage shader and pass --- .../viewer/src/modules/SpeckleRenderer.ts | 1 + packages/viewer/src/modules/Viewer.ts | 2 +- .../materials/shaders/speckle-sao-frag.ts | 2 +- .../speckle-static-ao-generate-frag.ts | 211 ++++++++++++++++++ .../speckle-static-ao-generate-vert.ts | 6 + .../src/modules/pipeline/ApplySAOPass.ts | 25 ++- .../viewer/src/modules/pipeline/Pipeline.ts | 7 +- .../modules/pipeline/SpeckleDynamicSAOPass.ts | 8 +- .../pipeline/SpeckleStaticAOGeneratePass.ts | 105 +++++++++ 9 files changed, 349 insertions(+), 18 deletions(-) create mode 100644 packages/viewer/src/modules/materials/shaders/speckle-static-ao-generate-frag.ts create mode 100644 packages/viewer/src/modules/materials/shaders/speckle-static-ao-generate-vert.ts create mode 100644 packages/viewer/src/modules/pipeline/SpeckleStaticAOGeneratePass.ts diff --git a/packages/viewer/src/modules/SpeckleRenderer.ts b/packages/viewer/src/modules/SpeckleRenderer.ts index 7c137d65a..de2ba8e69 100644 --- a/packages/viewer/src/modules/SpeckleRenderer.ts +++ b/packages/viewer/src/modules/SpeckleRenderer.ts @@ -318,6 +318,7 @@ export default class SpeckleRenderer { // this.lastPolar = currentPolar // this.lastDistance = currentDistance // this.lastTarget.copy(currentTarget) + this.viewer.needsRender = this.pipeline.needsRender } public render(camera: Camera) { diff --git a/packages/viewer/src/modules/Viewer.ts b/packages/viewer/src/modules/Viewer.ts index 9fbbd82a2..c27981b0d 100644 --- a/packages/viewer/src/modules/Viewer.ts +++ b/packages/viewer/src/modules/Viewer.ts @@ -155,7 +155,7 @@ export class Viewer extends EventEmitter implements IViewer { private render() { if (this.needsRender) { this.speckleRenderer.render(this.cameraHandler.activeCam.camera) - // this._needsRender = false + this._needsRender = false } } diff --git a/packages/viewer/src/modules/materials/shaders/speckle-sao-frag.ts b/packages/viewer/src/modules/materials/shaders/speckle-sao-frag.ts index b7c283856..53dc6b550 100644 --- a/packages/viewer/src/modules/materials/shaders/speckle-sao-frag.ts +++ b/packages/viewer/src/modules/materials/shaders/speckle-sao-frag.ts @@ -211,6 +211,6 @@ export const speckleSaoFrag = /* glsl */ ` vec3 viewPosition = getViewPosition( vUv, centerDepth, centerViewZ ); float ambientOcclusion = getAmbientOcclusion( viewPosition, centerDepth ); gl_FragColor = getDefaultColor( vUv ); - gl_FragColor.xyz *= 1.0 - ambientOcclusion; + gl_FragColor.xyz *= 1. - ambientOcclusion; // gl_FragColor.xyz = depth_cross(vUv, viewPosition) * 0.5 + 0.5; }` diff --git a/packages/viewer/src/modules/materials/shaders/speckle-static-ao-generate-frag.ts b/packages/viewer/src/modules/materials/shaders/speckle-static-ao-generate-frag.ts new file mode 100644 index 000000000..14b3b5fe2 --- /dev/null +++ b/packages/viewer/src/modules/materials/shaders/speckle-static-ao-generate-frag.ts @@ -0,0 +1,211 @@ +export const speckleStaticAoGenerateFrag = /* glsl */ ` + #include + varying vec2 vUv; + uniform sampler2D tDepth; + uniform sampler2D tNormal; + uniform vec2 size; + + uniform float cameraNear; + uniform float cameraFar; + uniform mat4 cameraProjectionMatrix; + uniform mat4 cameraInverseProjectionMatrix; + + uniform float scale; + uniform float intensity; + uniform float bias; + uniform float kernelRadius; + uniform float minResolution; + uniform float frameIndex; + + #define NUM_SAMPLES 16 + #define SPIRAL_TURNS 2 + #define NUM_FRAMES 16 + + // RGBA depth + #include + vec4 getDefaultColor( const in vec2 screenPosition ) { + return vec4( 1.0 ); + } + + float getDepth( const in vec2 screenPosition ) { + return unpackRGBAToDepth( texture2D( tDepth, screenPosition ) ); + } + + float getViewZ( const in float depth ) { + #if PERSPECTIVE_CAMERA == 1 + return perspectiveDepthToViewZ( depth, cameraNear, cameraFar ); + #else + return orthographicDepthToViewZ( depth, cameraNear, cameraFar ); + #endif + } + + vec3 getViewPosition( const in vec2 screenPosition, const in float depth, const in float viewZ ) { + float clipW = cameraProjectionMatrix[2][3] * viewZ + cameraProjectionMatrix[3][3]; + vec4 clipPosition = vec4( ( vec3( screenPosition, depth ) - 0.5 ) * 2.0, 1.0 ); + clipPosition *= clipW; // unprojection. + return ( cameraInverseProjectionMatrix * clipPosition ).xyz; + } + + //https://wickedengine.net/2019/09/22/improved-normal-reconstruction-from-depth/ + vec3 viewNormalImproved(in vec2 uv, in vec3 origin) + { + highp vec2 dd = abs(vec2(1./size.x, 1./size.y)); + highp vec2 ddx = vec2(dd.x, 0.); + highp vec2 ddy = vec2(0., dd.y); + + float sampleDepth = getDepth( uv - ddy ); + float sampleViewZ = getViewZ( sampleDepth ); + highp vec3 top = getViewPosition( uv - ddy, sampleDepth, sampleViewZ ); + + sampleDepth = getDepth( uv + ddy ); + sampleViewZ = getViewZ( sampleDepth ); + highp vec3 bottom = getViewPosition( uv + ddy, sampleDepth, sampleViewZ ); + + highp vec3 center = origin; + + sampleDepth = getDepth( uv - ddx ); + sampleViewZ = getViewZ( sampleDepth ); + highp vec3 left = getViewPosition( uv - ddx, sampleDepth, sampleViewZ ); + + sampleDepth = getDepth( uv + ddx ); + sampleViewZ = getViewZ( sampleDepth ); + highp vec3 right = getViewPosition( uv + ddx, sampleDepth, sampleViewZ ); + + // get the difference between the current and each offset position + vec3 l = center - left; + vec3 r = right - center; + vec3 d = center - top; + vec3 u = bottom - center; + + // pick horizontal and vertical diff with the smallest z difference + vec3 hDeriv = abs(l.z) < abs(r.z) ? l : r; + vec3 vDeriv = abs(d.z) < abs(u.z) ? d : u; + + // get view space normal from the cross product of the two smallest offsets + vec3 viewNormal = normalize(cross(hDeriv, vDeriv)); + + return viewNormal; + } + + vec3 viewNormalAccurate(in vec2 uv, in vec3 origin, in float centerDepth) { + highp vec2 dd = abs(vec2(1./size.x, 1./size.y)); + highp vec2 ddx = vec2(dd.x, 0.); + highp vec2 ddy = vec2(0., dd.y); + + float sampleDepth = getDepth( uv - ddy ); + float sampleViewZ = getViewZ( sampleDepth ); + highp vec3 top = getViewPosition( uv - ddy, sampleDepth, sampleViewZ ); + + sampleDepth = getDepth( uv + ddy ); + sampleViewZ = getViewZ( sampleDepth ); + highp vec3 bottom = getViewPosition( uv + ddy, sampleDepth, sampleViewZ ); + + highp vec3 center = origin; + + sampleDepth = getDepth( uv - ddx ); + sampleViewZ = getViewZ( sampleDepth ); + highp vec3 left = getViewPosition( uv - ddx, sampleDepth, sampleViewZ ); + + sampleDepth = getDepth( uv + ddx ); + sampleViewZ = getViewZ( sampleDepth ); + highp vec3 right = getViewPosition( uv + ddx, sampleDepth, sampleViewZ ); + + // get the difference between the current and each offset position + vec3 l = center - left; + vec3 r = right - center; + vec3 d = center - top; + vec3 u = bottom - center; + + // get depth values at 1 & 2 pixels offsets from current along the horizontal axis + vec4 H = vec4( + getDepth(uv - ddx), + getDepth(uv + ddx), + getDepth(uv - 2. * ddx), + getDepth(uv + 2. * ddx) + ); + + // get depth values at 1 & 2 pixels offsets from current along the vertical axis + vec4 V = vec4( + getDepth(uv - ddy), + getDepth(uv + ddy), + getDepth(uv - 2. * ddy), + getDepth(uv + 2. * ddy) + ); + + // current pixel's depth difference from slope of offset depth samples + // differs from original article because we're using non-linear depth values + // see article's comments + vec2 he = abs((2. * H.xy - H.zw) - centerDepth); + vec2 ve = abs((2. * V.xy - V.zw) - centerDepth); + + // pick horizontal and vertical diff with the smallest depth difference from slopes + vec3 hDeriv = he.x < he.y ? l : r; + vec3 vDeriv = ve.x < ve.y ? d : u; + + // get view space normal from the cross product of the best derivatives + vec3 viewNormal = normalize(cross(hDeriv, vDeriv)); + + return viewNormal; + + } + + vec3 getViewNormal( const in vec3 viewPosition, const in vec2 screenPosition, in float centerDepth ) { + #if NORMAL_TEXTURE == 1 + return unpackRGBToNormal( texture2D( tNormal, screenPosition ).xyz ); + #elif IMPROVED_NORMAL_RECONSTRUCTION == 1 + return viewNormalImproved(screenPosition, viewPosition); + #elif ACCURATE_NORMAL_RECONSTRUCTION == 1 + return viewNormalAccurate(screenPosition, viewPosition, centerDepth); + #else + return normalize( cross( dFdx( viewPosition ), dFdy( viewPosition ) ) ); + #endif + } + + float scaleDividedByCameraFar; + float minResolutionMultipliedByCameraFar; + // moving costly divides into consts + const float INV_NUM_SAMPLES = 1.0 / float( NUM_SAMPLES ); + const float offset = PI2 / NUM_FRAMES + + float getAmbientOcclusion( const in vec3 centerViewPosition, in float centerDepth ) { + #if AO_ESTIMATOR == 0 + // precompute some variables require in getOcclusion. + scaleDividedByCameraFar = scale / cameraFar; + minResolutionMultipliedByCameraFar = minResolution * cameraFar; + vec3 centerViewNormal = getViewNormal( centerViewPosition, vUv, centerDepth ); + // jsfiddle that shows sample pattern: https://jsfiddle.net/TenHands/jun67k9y/7/ + float occlusionSum = 0.0; + float weightSum = 0.0; + for( int i = 0; i < NUM_SAMPLES; i ++ ) { + float alpha = ( i + 1. ) / NUM_SAMPLES; + float angle = SPIRAL_TURNS * alpha; + vec2 radius = (kernelRadius / size) * Math.pow( alpha, 1.1 ); + vec2 sampleUv = vUv + vec2( cos( angle + frameIndex * offset ), sin( angle + frameIndex * offset ) ) * radius; + + float sampleDepth = getDepth( sampleUv ); + if( sampleDepth >= ( 1.0 - EPSILON ) ) { + continue; + } + float sampleViewZ = getViewZ( sampleDepth ); + vec3 sampleViewPosition = getViewPosition( sampleUv, sampleDepth, sampleViewZ ); + vec3 viewDelta = sampleViewPosition - centerViewPosition; + float viewDistance = length( viewDelta ); + float scaledScreenDistance = scaleDividedByCameraFar * viewDistance; + occlusionSum += max(0.0, (dot(centerViewNormal, viewDelta) - minResolutionMultipliedByCameraFar) / scaledScreenDistance - bias) / (1.0 + pow2( scaledScreenDistance ) ); + weightSum += 1.0; + } + if( weightSum == 0.0 ) discard; + return occlusionSum * ( intensity / weightSum ); + #endif + } + void main() { + float centerDepth = getDepth( vUv ); + if( centerDepth >= ( 1.0 - EPSILON ) ) { + discard; + } + float centerViewZ = getViewZ( centerDepth ); + vec3 viewPosition = getViewPosition( vUv, centerDepth, centerViewZ ); + float ambientOcclusion = getAmbientOcclusion( viewPosition, centerDepth ); + gl_FragColor = getDefaultColor( vUv ); + gl_FragColor.xyz *= 1. - ambientOcclusion; + }` diff --git a/packages/viewer/src/modules/materials/shaders/speckle-static-ao-generate-vert.ts b/packages/viewer/src/modules/materials/shaders/speckle-static-ao-generate-vert.ts new file mode 100644 index 000000000..72263082b --- /dev/null +++ b/packages/viewer/src/modules/materials/shaders/speckle-static-ao-generate-vert.ts @@ -0,0 +1,6 @@ +export const speckleStaticAoGenerateVert = /* glsl */ ` + varying vec2 vUv; + void main() { + vUv = uv; + gl_Position = projectionMatrix * modelViewMatrix * vec4( position, 1.0 ); + }` diff --git a/packages/viewer/src/modules/pipeline/ApplySAOPass.ts b/packages/viewer/src/modules/pipeline/ApplySAOPass.ts index 92c2ad7d7..2a0d5b49e 100644 --- a/packages/viewer/src/modules/pipeline/ApplySAOPass.ts +++ b/packages/viewer/src/modules/pipeline/ApplySAOPass.ts @@ -14,7 +14,7 @@ import { CopyShader } from 'three/examples/jsm/shaders/CopyShader.js' export class ApplySAOPass extends Pass { private fsQuad: FullScreenQuad - private materialCopy: ShaderMaterial + public materialCopy: ShaderMaterial constructor(srcSao: Texture) { super() @@ -24,15 +24,22 @@ export class ApplySAOPass extends Pass { fragmentShader: CopyShader.fragmentShader, blending: NoBlending }) - // this.materialCopy.transparent = true - // this.materialCopy.depthTest = false - // this.materialCopy.depthWrite = false + this.materialCopy.transparent = true + this.materialCopy.depthTest = false + this.materialCopy.depthWrite = false + this.materialCopy.blending = CustomBlending + this.materialCopy.blendSrc = DstColorFactor + this.materialCopy.blendDst = ZeroFactor + this.materialCopy.blendEquation = AddEquation + this.materialCopy.blendSrcAlpha = DstAlphaFactor + this.materialCopy.blendDstAlpha = ZeroFactor + this.materialCopy.blendEquationAlpha = AddEquation // this.materialCopy.blending = CustomBlending - // this.materialCopy.blendSrc = DstColorFactor - // this.materialCopy.blendDst = ZeroFactor - // this.materialCopy.blendEquation = AddEquation - // this.materialCopy.blendSrcAlpha = DstAlphaFactor - // this.materialCopy.blendDstAlpha = ZeroFactor + // this.materialCopy.blendSrc = OneFactor + // this.materialCopy.blendDst = OneFactor + // this.materialCopy.blendEquation = ReverseSubtractEquation + // this.materialCopy.blendSrcAlpha = OneFactor + // this.materialCopy.blendDstAlpha = OneFactor // this.materialCopy.blendEquationAlpha = AddEquation this.materialCopy.uniforms['tDiffuse'].value = srcSao this.materialCopy.needsUpdate = true diff --git a/packages/viewer/src/modules/pipeline/Pipeline.ts b/packages/viewer/src/modules/pipeline/Pipeline.ts index 02af84781..b15b49856 100644 --- a/packages/viewer/src/modules/pipeline/Pipeline.ts +++ b/packages/viewer/src/modules/pipeline/Pipeline.ts @@ -39,6 +39,7 @@ export class Pipeline { private saoPass: SpeckleDynamicSAOPass = null private applySaoPass: ApplySAOPass = null private drawingSize: Vector2 = new Vector2() + public needsRender = true public set pipelineOptions(options: PipelineOptions) { Object.assign(this._pipelineOptions, options) @@ -69,7 +70,6 @@ export class Pipeline { this.composer.addPass(this.saoPass) this.renderPass = new RenderPass(scene, camera) this.renderPass.renderToScreen = true - this.renderPass.enabled = false this.composer.addPass(this.renderPass) this.applySaoPass = new ApplySAOPass(this.saoPass.saoRenderTarget.texture) this.applySaoPass.renderToScreen = true @@ -85,6 +85,7 @@ export class Pipeline { this._renderer.getDrawingBufferSize(this.drawingSize) if (this.drawingSize.length() === 0) return + this._renderer.clear(true) this.renderPass.scene = scene this.renderPass.camera = camera this.saoPass.scene = scene @@ -97,10 +98,10 @@ export class Pipeline { } public onStationaryBegin() { - this.saoPass.accumulate = true + this.needsRender = true } public onStationaryEnd() { - this.saoPass.accumulate = false + this.needsRender = true } } diff --git a/packages/viewer/src/modules/pipeline/SpeckleDynamicSAOPass.ts b/packages/viewer/src/modules/pipeline/SpeckleDynamicSAOPass.ts index de888efcc..b47fe07e7 100644 --- a/packages/viewer/src/modules/pipeline/SpeckleDynamicSAOPass.ts +++ b/packages/viewer/src/modules/pipeline/SpeckleDynamicSAOPass.ts @@ -1,4 +1,5 @@ import { + AdditiveBlending, Camera, DoubleSide, NoBlending, @@ -36,7 +37,6 @@ export class SpeckleDynamicSAOPass extends SAOPass { private prevNumSamples private batcher: Batcher = null private normalsType: NormalsType = NormalsType.IMPROVED - public accumulate = false public set normalsRendering(type: NormalsType) { this.normalsType = type @@ -131,7 +131,7 @@ export class SpeckleDynamicSAOPass extends SAOPass { renderer.setRenderTarget(this.depthRenderTarget) renderer.clear() - + this.saoMaterial.blending = AdditiveBlending this.saoMaterial.uniforms['bias'].value = this.params.saoBias this.saoMaterial.uniforms['intensity'].value = this.params.saoIntensity this.saoMaterial.uniforms['scale'].value = this.params.saoScale @@ -221,7 +221,7 @@ export class SpeckleDynamicSAOPass extends SAOPass { this.batcher.applyVisibility(restoreVisibility) // Rendering SAO texture - this.renderPass(renderer, this.saoMaterial, this.saoRenderTarget, 0xffffff, 1.0) + this.renderPass(renderer, this.saoMaterial, this.saoRenderTarget, 0x000000, 1.0) // Blurring SAO texture if (this.params.saoBlur) { @@ -252,7 +252,7 @@ export class SpeckleDynamicSAOPass extends SAOPass { // setup pass state renderer.autoClear = false - if (clearColor !== undefined && clearColor !== null && !this.accumulate) { + if (clearColor !== undefined && clearColor !== null) { renderer.setClearColor(clearColor) renderer.setClearAlpha(clearAlpha || 0.0) renderer.clear() diff --git a/packages/viewer/src/modules/pipeline/SpeckleStaticAOGeneratePass.ts b/packages/viewer/src/modules/pipeline/SpeckleStaticAOGeneratePass.ts new file mode 100644 index 000000000..473874cf2 --- /dev/null +++ b/packages/viewer/src/modules/pipeline/SpeckleStaticAOGeneratePass.ts @@ -0,0 +1,105 @@ +import { + Camera, + Matrix4, + NoBlending, + OrthographicCamera, + PerspectiveCamera, + ShaderMaterial, + Texture, + Vector2, + WebGLRenderTarget +} from 'three' +import { Pass } from 'three/examples/jsm/postprocessing/Pass' + +import Batcher from '../batching/Batcher' +import { speckleStaticAoGenerateVert } from '../materials/shaders/speckle-static-ao-generate-vert' +import { speckleStaticAoGenerateFrag } from '../materials/shaders/speckle-static-ao-generate-frag' +/** + * SAO implementation inspired from bhouston previous SAO work + */ + +export class SpeckleStaticAOGeneratePass extends Pass { + private batcher: Batcher = null + private aoMaterial: ShaderMaterial = null + private _depthTexture: Texture + private _normalTexture: Texture + private _generationBuffer: WebGLRenderTarget + private _accumulationBuffer: WebGLRenderTarget + private _renderIndex = 0 + + public set depthTexture(value: Texture) { + this._depthTexture = value + } + + public set normalTexture(value: Texture) { + this._normalTexture = value + } + + constructor(batcher: Batcher) { + super() + + this.batcher = batcher + this._generationBuffer = new WebGLRenderTarget(256, 256) + this._accumulationBuffer = new WebGLRenderTarget(256, 256) + const aoDefines = { + AO_ESTIMATOR: 0 + } + + this.aoMaterial = new ShaderMaterial({ + defines: aoDefines, + fragmentShader: speckleStaticAoGenerateVert, + vertexShader: speckleStaticAoGenerateFrag, + uniforms: { + tDepth: { value: null }, + tNormal: { value: null }, + size: { value: new Vector2(512, 512) }, + + cameraNear: { value: 1 }, + cameraFar: { value: 100 }, + cameraProjectionMatrix: { value: new Matrix4() }, + cameraInverseProjectionMatrix: { value: new Matrix4() }, + + scale: { value: 1.0 }, + intensity: { value: 0.1 }, + bias: { value: 0.5 }, + + minResolution: { value: 0.0 }, + kernelRadius: { value: 100.0 }, + randomSeed: { value: 0.0 } + } + }) + + this.aoMaterial.extensions.derivatives = true + this.aoMaterial.uniforms['size'].value.set(256, 256) + this.aoMaterial.blending = NoBlending + } + + public update(camera: Camera) { + this.aoMaterial.uniforms['cameraNear'].value = ( + camera as PerspectiveCamera | OrthographicCamera + ).near + this.aoMaterial.uniforms['cameraFar'].value = ( + camera as PerspectiveCamera | OrthographicCamera + ).far + this.aoMaterial.uniforms['cameraInverseProjectionMatrix'].value.copy( + camera.projectionMatrixInverse + ) + this.aoMaterial.uniforms['cameraProjectionMatrix'].value.copy( + camera.projectionMatrix + ) + } + + public render(renderer, writeBuffer, readBuffer) { + renderer + writeBuffer + readBuffer + } + + public setSize(width: number, height: number) { + this._generationBuffer.setSize(width, height) + this._accumulationBuffer.setSize(width, height) + + this.aoMaterial.uniforms['size'].value.set(width, height) + this.aoMaterial.needsUpdate = true + } +} From e07eb1fd346875a508e8bd71a9c81b95a9c0fac6 Mon Sep 17 00:00:00 2001 From: AlexandruPopovici Date: Fri, 7 Oct 2022 18:14:39 +0300 Subject: [PATCH 03/54] Continued with progressive SAO. Added the accumulation pass --- .../speckle-static-ao-accumulate-frag.ts | 11 +++ .../speckle-static-ao-accumulate-vert.ts | 6 ++ .../speckle-static-ao-generate-frag.ts | 17 ++-- .../src/modules/pipeline/ApplySAOPass.ts | 33 +++----- .../viewer/src/modules/pipeline/Pipeline.ts | 11 ++- .../pipeline/SpeckleStaticAOGeneratePass.ts | 83 +++++++++++++++++-- 6 files changed, 125 insertions(+), 36 deletions(-) create mode 100644 packages/viewer/src/modules/materials/shaders/speckle-static-ao-accumulate-frag.ts create mode 100644 packages/viewer/src/modules/materials/shaders/speckle-static-ao-accumulate-vert.ts diff --git a/packages/viewer/src/modules/materials/shaders/speckle-static-ao-accumulate-frag.ts b/packages/viewer/src/modules/materials/shaders/speckle-static-ao-accumulate-frag.ts new file mode 100644 index 000000000..1a17cfdf5 --- /dev/null +++ b/packages/viewer/src/modules/materials/shaders/speckle-static-ao-accumulate-frag.ts @@ -0,0 +1,11 @@ +export const speckleStaticAoAccumulateFrag = /* glsl */ ` + uniform float opacity; + uniform sampler2D tDiffuse; + varying vec2 vUv; + #define NUM_FRAMES 16 + + void main() { + vec4 frameSample = texture2D( tDiffuse, vUv ); + gl_FragColor.xyz = frameSample.rgb * 1./float(NUM_FRAMES); + gl_FragColor.a = 1.;//*= opacity; + }` diff --git a/packages/viewer/src/modules/materials/shaders/speckle-static-ao-accumulate-vert.ts b/packages/viewer/src/modules/materials/shaders/speckle-static-ao-accumulate-vert.ts new file mode 100644 index 000000000..30ad72282 --- /dev/null +++ b/packages/viewer/src/modules/materials/shaders/speckle-static-ao-accumulate-vert.ts @@ -0,0 +1,6 @@ +export const speckleStaticAoAccumulateVert = /* glsl */ ` + varying vec2 vUv; + void main() { + vUv = uv; + gl_Position = projectionMatrix * modelViewMatrix * vec4( position, 1.0 ); + }` diff --git a/packages/viewer/src/modules/materials/shaders/speckle-static-ao-generate-frag.ts b/packages/viewer/src/modules/materials/shaders/speckle-static-ao-generate-frag.ts index 14b3b5fe2..259dfaf34 100644 --- a/packages/viewer/src/modules/materials/shaders/speckle-static-ao-generate-frag.ts +++ b/packages/viewer/src/modules/materials/shaders/speckle-static-ao-generate-frag.ts @@ -20,6 +20,12 @@ export const speckleStaticAoGenerateFrag = /* glsl */ ` #define NUM_SAMPLES 16 #define SPIRAL_TURNS 2 #define NUM_FRAMES 16 + + #define NORMAL_TEXTURE 0 + #define IMPROVED_NORMAL_RECONSTRUCTION 0 + #define ACCURATE_NORMAL_RECONSTRUCTION 1 + + #define AO_ESTIMATOR 0 // RGBA depth #include @@ -165,7 +171,7 @@ export const speckleStaticAoGenerateFrag = /* glsl */ ` float minResolutionMultipliedByCameraFar; // moving costly divides into consts const float INV_NUM_SAMPLES = 1.0 / float( NUM_SAMPLES ); - const float offset = PI2 / NUM_FRAMES + const float offset = PI2 / float(NUM_FRAMES); float getAmbientOcclusion( const in vec3 centerViewPosition, in float centerDepth ) { #if AO_ESTIMATOR == 0 @@ -177,9 +183,9 @@ export const speckleStaticAoGenerateFrag = /* glsl */ ` float occlusionSum = 0.0; float weightSum = 0.0; for( int i = 0; i < NUM_SAMPLES; i ++ ) { - float alpha = ( i + 1. ) / NUM_SAMPLES; - float angle = SPIRAL_TURNS * alpha; - vec2 radius = (kernelRadius / size) * Math.pow( alpha, 1.1 ); + float alpha = ( float(i) + 1. ) / float(NUM_SAMPLES); + float angle = float(SPIRAL_TURNS) * alpha; + vec2 radius = (kernelRadius / size) * pow( alpha, 1.1 ); vec2 sampleUv = vUv + vec2( cos( angle + frameIndex * offset ), sin( angle + frameIndex * offset ) ) * radius; float sampleDepth = getDepth( sampleUv ); @@ -207,5 +213,6 @@ export const speckleStaticAoGenerateFrag = /* glsl */ ` vec3 viewPosition = getViewPosition( vUv, centerDepth, centerViewZ ); float ambientOcclusion = getAmbientOcclusion( viewPosition, centerDepth ); gl_FragColor = getDefaultColor( vUv ); - gl_FragColor.xyz *= 1. - ambientOcclusion; + gl_FragColor.xyz *= ambientOcclusion; + // gl_FragColor.xyz = getViewNormal(viewPosition, vUv, centerDepth); }` diff --git a/packages/viewer/src/modules/pipeline/ApplySAOPass.ts b/packages/viewer/src/modules/pipeline/ApplySAOPass.ts index 2a0d5b49e..1861b9d11 100644 --- a/packages/viewer/src/modules/pipeline/ApplySAOPass.ts +++ b/packages/viewer/src/modules/pipeline/ApplySAOPass.ts @@ -1,14 +1,4 @@ -import { - AddEquation, - CustomBlending, - DstAlphaFactor, - DstColorFactor, - NoBlending, - ShaderMaterial, - Texture, - UniformsUtils, - ZeroFactor -} from 'three' +import { NoBlending, ShaderMaterial, Texture, UniformsUtils } from 'three' import { FullScreenQuad, Pass } from 'three/examples/jsm/postprocessing/Pass' import { CopyShader } from 'three/examples/jsm/shaders/CopyShader.js' @@ -24,16 +14,17 @@ export class ApplySAOPass extends Pass { fragmentShader: CopyShader.fragmentShader, blending: NoBlending }) - this.materialCopy.transparent = true - this.materialCopy.depthTest = false - this.materialCopy.depthWrite = false - this.materialCopy.blending = CustomBlending - this.materialCopy.blendSrc = DstColorFactor - this.materialCopy.blendDst = ZeroFactor - this.materialCopy.blendEquation = AddEquation - this.materialCopy.blendSrcAlpha = DstAlphaFactor - this.materialCopy.blendDstAlpha = ZeroFactor - this.materialCopy.blendEquationAlpha = AddEquation + // this.materialCopy.transparent = true + // this.materialCopy.depthTest = false + // this.materialCopy.depthWrite = false + // this.materialCopy.blending = CustomBlending + // this.materialCopy.blendSrc = DstColorFactor + // this.materialCopy.blendDst = ZeroFactor + // this.materialCopy.blendEquation = AddEquation + // this.materialCopy.blendSrcAlpha = DstAlphaFactor + // this.materialCopy.blendDstAlpha = ZeroFactor + // this.materialCopy.blendEquationAlpha = AddEquation + // this.materialCopy.blending = CustomBlending // this.materialCopy.blendSrc = OneFactor // this.materialCopy.blendDst = OneFactor diff --git a/packages/viewer/src/modules/pipeline/Pipeline.ts b/packages/viewer/src/modules/pipeline/Pipeline.ts index b15b49856..9209f67d8 100644 --- a/packages/viewer/src/modules/pipeline/Pipeline.ts +++ b/packages/viewer/src/modules/pipeline/Pipeline.ts @@ -5,6 +5,7 @@ import { SAOPassParams } from 'three/examples/jsm/postprocessing/SAOPass.js' import Batcher from '../batching/Batcher' import { ApplySAOPass } from './ApplySAOPass' import { NormalsType, SpeckleDynamicSAOPass } from './SpeckleDynamicSAOPass' +import { SpeckleStaticAOGeneratePass } from './SpeckleStaticAOGeneratePass' export interface PipelineOptions { saoEnabled?: boolean @@ -38,6 +39,7 @@ export class Pipeline { private renderPass: RenderPass = null private saoPass: SpeckleDynamicSAOPass = null private applySaoPass: ApplySAOPass = null + private staticAOGenerationPass: SpeckleStaticAOGeneratePass = null private drawingSize: Vector2 = new Vector2() public needsRender = true @@ -67,11 +69,17 @@ export class Pipeline { false, NormalsType.IMPROVED ) + this.staticAOGenerationPass = new SpeckleStaticAOGeneratePass(this._batcher) + this.staticAOGenerationPass.depthTexture = this.saoPass.depthRenderTarget.texture this.composer.addPass(this.saoPass) this.renderPass = new RenderPass(scene, camera) this.renderPass.renderToScreen = true + this.renderPass.enabled = false this.composer.addPass(this.renderPass) - this.applySaoPass = new ApplySAOPass(this.saoPass.saoRenderTarget.texture) + this.composer.addPass(this.staticAOGenerationPass) + this.applySaoPass = new ApplySAOPass( + this.staticAOGenerationPass.outputTexture.texture + ) this.applySaoPass.renderToScreen = true this.composer.addPass(this.applySaoPass) } @@ -90,6 +98,7 @@ export class Pipeline { this.renderPass.camera = camera this.saoPass.scene = scene this.saoPass.camera = camera + this.staticAOGenerationPass.update(camera) this.composer.render() } diff --git a/packages/viewer/src/modules/pipeline/SpeckleStaticAOGeneratePass.ts b/packages/viewer/src/modules/pipeline/SpeckleStaticAOGeneratePass.ts index 473874cf2..6cd2569cc 100644 --- a/packages/viewer/src/modules/pipeline/SpeckleStaticAOGeneratePass.ts +++ b/packages/viewer/src/modules/pipeline/SpeckleStaticAOGeneratePass.ts @@ -1,5 +1,6 @@ import { Camera, + Color, Matrix4, NoBlending, OrthographicCamera, @@ -7,13 +8,16 @@ import { ShaderMaterial, Texture, Vector2, + WebGLRenderer, WebGLRenderTarget } from 'three' -import { Pass } from 'three/examples/jsm/postprocessing/Pass' +import { FullScreenQuad, Pass } from 'three/examples/jsm/postprocessing/Pass' import Batcher from '../batching/Batcher' import { speckleStaticAoGenerateVert } from '../materials/shaders/speckle-static-ao-generate-vert' import { speckleStaticAoGenerateFrag } from '../materials/shaders/speckle-static-ao-generate-frag' +import { speckleStaticAoAccumulateVert } from '../materials/shaders/speckle-static-ao-accumulate-vert' +import { speckleStaticAoAccumulateFrag } from '../materials/shaders/speckle-static-ao-accumulate-frag' /** * SAO implementation inspired from bhouston previous SAO work */ @@ -21,20 +25,27 @@ import { speckleStaticAoGenerateFrag } from '../materials/shaders/speckle-static export class SpeckleStaticAOGeneratePass extends Pass { private batcher: Batcher = null private aoMaterial: ShaderMaterial = null + private accumulateMaterial: ShaderMaterial = null private _depthTexture: Texture private _normalTexture: Texture private _generationBuffer: WebGLRenderTarget private _accumulationBuffer: WebGLRenderTarget - private _renderIndex = 0 + private fsQuad: FullScreenQuad public set depthTexture(value: Texture) { this._depthTexture = value + this.aoMaterial.uniforms['tDepth'].value = value + this.aoMaterial.needsUpdate = true } public set normalTexture(value: Texture) { this._normalTexture = value } + public get outputTexture() { + return this._accumulationBuffer + } + constructor(batcher: Batcher) { super() @@ -47,8 +58,8 @@ export class SpeckleStaticAOGeneratePass extends Pass { this.aoMaterial = new ShaderMaterial({ defines: aoDefines, - fragmentShader: speckleStaticAoGenerateVert, - vertexShader: speckleStaticAoGenerateFrag, + fragmentShader: speckleStaticAoGenerateFrag, + vertexShader: speckleStaticAoGenerateVert, uniforms: { tDepth: { value: null }, tNormal: { value: null }, @@ -60,18 +71,34 @@ export class SpeckleStaticAOGeneratePass extends Pass { cameraInverseProjectionMatrix: { value: new Matrix4() }, scale: { value: 1.0 }, - intensity: { value: 0.1 }, - bias: { value: 0.5 }, + intensity: { value: 1 }, + bias: { value: 0 }, minResolution: { value: 0.0 }, - kernelRadius: { value: 100.0 }, - randomSeed: { value: 0.0 } + kernelRadius: { value: 10.0 }, + randomSeed: { value: 0.0 }, + + frameIndex: { value: 0 } } }) this.aoMaterial.extensions.derivatives = true this.aoMaterial.uniforms['size'].value.set(256, 256) this.aoMaterial.blending = NoBlending + + this.accumulateMaterial = new ShaderMaterial({ + defines: {}, + fragmentShader: speckleStaticAoAccumulateFrag, + vertexShader: speckleStaticAoAccumulateVert, + uniforms: { + tDiffuse: { value: null }, + opacity: { value: 1 } + } + }) + this.accumulateMaterial.uniforms['tDiffuse'].value = this._generationBuffer.texture + this.accumulateMaterial.blending = NoBlending + + this.fsQuad = new FullScreenQuad(this.aoMaterial) } public update(camera: Camera) { @@ -87,12 +114,50 @@ export class SpeckleStaticAOGeneratePass extends Pass { this.aoMaterial.uniforms['cameraProjectionMatrix'].value.copy( camera.projectionMatrix ) + this.aoMaterial.uniforms['scale'].value = ( + camera as PerspectiveCamera | OrthographicCamera + ).far + this.aoMaterial.defines['PERSPECTIVE_CAMERA'] = (camera as PerspectiveCamera) + .isPerspectiveCamera + ? 1 + : 0 + this.aoMaterial.needsUpdate = true } public render(renderer, writeBuffer, readBuffer) { - renderer writeBuffer readBuffer + + // save original state + const originalClearColor = new Color() + renderer.getClearColor(originalClearColor) + const originalClearAlpha = renderer.getClearAlpha() + const originalAutoClear = renderer.autoClear + + this.renderFrame(renderer, 0) + // restore original state + renderer.autoClear = originalAutoClear + renderer.setClearColor(originalClearColor) + renderer.setClearAlpha(originalClearAlpha) + } + + private renderFrame(renderer: WebGLRenderer, frameIndex: number) { + this.aoMaterial.uniforms['frameIndex'].value = frameIndex + + renderer.setRenderTarget(this._generationBuffer) + renderer.autoClear = false + renderer.setClearColor(0x000000) + renderer.setClearAlpha(1) + renderer.clear() + this.fsQuad.material = this.aoMaterial + this.fsQuad.render(renderer) + + renderer.setRenderTarget(this._accumulationBuffer) + renderer.setClearColor(0x000000) + renderer.setClearAlpha(1) + renderer.clear() + this.fsQuad.material = this.accumulateMaterial + this.fsQuad.render(renderer) } public setSize(width: number, height: number) { From 775be15c307c5815d0685f77bbced64ba45b21a3 Mon Sep 17 00:00:00 2001 From: AlexandruPopovici Date: Mon, 10 Oct 2022 12:29:45 +0300 Subject: [PATCH 04/54] Implemented progressive AO using the same estimator as for the dynamic one. --- packages/viewer-sandbox/src/main.ts | 4 +- .../viewer/src/modules/SpeckleRenderer.ts | 16 ++--- packages/viewer/src/modules/Viewer.ts | 5 +- .../speckle-static-ao-accumulate-frag.ts | 2 +- .../speckle-static-ao-generate-frag.ts | 1 + .../src/modules/pipeline/ApplySAOPass.ts | 40 +++++++++---- .../viewer/src/modules/pipeline/Pipeline.ts | 60 ++++++++++++++----- .../pipeline/SpeckleStaticAOGeneratePass.ts | 33 ++++++---- 8 files changed, 110 insertions(+), 51 deletions(-) diff --git a/packages/viewer-sandbox/src/main.ts b/packages/viewer-sandbox/src/main.ts index 067770572..09401f81a 100644 --- a/packages/viewer-sandbox/src/main.ts +++ b/packages/viewer-sandbox/src/main.ts @@ -82,7 +82,7 @@ sandbox.makeFilteringUI() await sandbox.loadUrl( // 'https://speckle.xyz/streams/da9e320dad/commits/5388ef24b8?c=%5B-7.66134,10.82932,6.41935,-0.07739,-13.88552,1.8697,0,1%5D' // Revit sample house (good for bim-like stuff with many display meshes) - 'https://speckle.xyz/streams/da9e320dad/commits/5388ef24b8' + // 'https://speckle.xyz/streams/da9e320dad/commits/5388ef24b8' // 'Super' heavy revit shit // 'https://speckle.xyz/streams/e6f9156405/commits/0694d53bb5' // Same sample revit house, local to dim's computer @@ -91,7 +91,7 @@ await sandbox.loadUrl( // IFC building (good for a tree based structure) // 'https://latest.speckle.dev/streams/92b620fb17/commits/2ebd336223' // IFC story, a subtree of the above - // 'https://latest.speckle.dev/streams/92b620fb17/objects/8247bbc53865b0e0cb5ee4e252e66216' + 'https://latest.speckle.dev/streams/92b620fb17/objects/8247bbc53865b0e0cb5ee4e252e66216' // Small scale lines // 'https://speckle.xyz/streams/638d3b1f83/commits/6025e2b546?c=%5B2.18058,-0.20814,9.67642,3.85491,5.05364,0,0,1%5D' // 'https://latest.speckle.dev/streams/3ed8357f29/commits/d10f2af1ce' diff --git a/packages/viewer/src/modules/SpeckleRenderer.ts b/packages/viewer/src/modules/SpeckleRenderer.ts index de2ba8e69..4c18d0ff8 100644 --- a/packages/viewer/src/modules/SpeckleRenderer.ts +++ b/packages/viewer/src/modules/SpeckleRenderer.ts @@ -178,13 +178,13 @@ export default class SpeckleRenderer { helpers.add(camHelper) } this.viewer.cameraHandler.controls.restThreshold = 0.001 - this.viewer.cameraHandler.controls.addEventListener('sleep', (event) => { + this.viewer.cameraHandler.controls.addEventListener('sleep', () => { this.pipeline.onStationaryBegin() }) - this.viewer.cameraHandler.controls.addEventListener('rest', (event) => { - this.pipeline.onStationaryBegin() - }) - this.viewer.cameraHandler.controls.addEventListener('wake', (event) => { + // this.viewer.cameraHandler.controls.addEventListener('rest', (event) => { + // this.pipeline.onStationaryBegin() + // }) + this.viewer.cameraHandler.controls.addEventListener('wake', () => { this.pipeline.onStationaryEnd() }) } @@ -318,12 +318,12 @@ export default class SpeckleRenderer { // this.lastPolar = currentPolar // this.lastDistance = currentDistance // this.lastTarget.copy(currentTarget) - this.viewer.needsRender = this.pipeline.needsRender } - public render(camera: Camera) { + public render(camera: Camera): boolean { this.batcher.render(this.renderer) - this.pipeline.render(this.scene, camera) + const needsRender = this.pipeline.render(this.scene, camera) + return needsRender // this.renderer.render(this.scene, camera) } diff --git a/packages/viewer/src/modules/Viewer.ts b/packages/viewer/src/modules/Viewer.ts index c27981b0d..8e816a122 100644 --- a/packages/viewer/src/modules/Viewer.ts +++ b/packages/viewer/src/modules/Viewer.ts @@ -154,8 +154,9 @@ export class Viewer extends EventEmitter implements IViewer { private render() { if (this.needsRender) { - this.speckleRenderer.render(this.cameraHandler.activeCam.camera) - this._needsRender = false + this._needsRender = this.speckleRenderer.render( + this.cameraHandler.activeCam.camera + ) } } diff --git a/packages/viewer/src/modules/materials/shaders/speckle-static-ao-accumulate-frag.ts b/packages/viewer/src/modules/materials/shaders/speckle-static-ao-accumulate-frag.ts index 1a17cfdf5..8b4298930 100644 --- a/packages/viewer/src/modules/materials/shaders/speckle-static-ao-accumulate-frag.ts +++ b/packages/viewer/src/modules/materials/shaders/speckle-static-ao-accumulate-frag.ts @@ -6,6 +6,6 @@ export const speckleStaticAoAccumulateFrag = /* glsl */ ` void main() { vec4 frameSample = texture2D( tDiffuse, vUv ); - gl_FragColor.xyz = frameSample.rgb * 1./float(NUM_FRAMES); + gl_FragColor.xyz = (frameSample.rgb) * 1./float(NUM_FRAMES); gl_FragColor.a = 1.;//*= opacity; }` diff --git a/packages/viewer/src/modules/materials/shaders/speckle-static-ao-generate-frag.ts b/packages/viewer/src/modules/materials/shaders/speckle-static-ao-generate-frag.ts index 259dfaf34..ece3651aa 100644 --- a/packages/viewer/src/modules/materials/shaders/speckle-static-ao-generate-frag.ts +++ b/packages/viewer/src/modules/materials/shaders/speckle-static-ao-generate-frag.ts @@ -214,5 +214,6 @@ export const speckleStaticAoGenerateFrag = /* glsl */ ` float ambientOcclusion = getAmbientOcclusion( viewPosition, centerDepth ); gl_FragColor = getDefaultColor( vUv ); gl_FragColor.xyz *= ambientOcclusion; + gl_FragColor.a = 1.;//1. / float(NUM_FRAMES); // gl_FragColor.xyz = getViewNormal(viewPosition, vUv, centerDepth); }` diff --git a/packages/viewer/src/modules/pipeline/ApplySAOPass.ts b/packages/viewer/src/modules/pipeline/ApplySAOPass.ts index 1861b9d11..65b2cf55e 100644 --- a/packages/viewer/src/modules/pipeline/ApplySAOPass.ts +++ b/packages/viewer/src/modules/pipeline/ApplySAOPass.ts @@ -1,4 +1,14 @@ -import { NoBlending, ShaderMaterial, Texture, UniformsUtils } from 'three' +import { + AddEquation, + CustomBlending, + DstAlphaFactor, + DstColorFactor, + NoBlending, + ShaderMaterial, + Texture, + UniformsUtils, + ZeroFactor +} from 'three' import { FullScreenQuad, Pass } from 'three/examples/jsm/postprocessing/Pass' import { CopyShader } from 'three/examples/jsm/shaders/CopyShader.js' @@ -6,7 +16,7 @@ export class ApplySAOPass extends Pass { private fsQuad: FullScreenQuad public materialCopy: ShaderMaterial - constructor(srcSao: Texture) { + constructor() { super() this.materialCopy = new ShaderMaterial({ uniforms: UniformsUtils.clone(CopyShader.uniforms), @@ -14,16 +24,16 @@ export class ApplySAOPass extends Pass { fragmentShader: CopyShader.fragmentShader, blending: NoBlending }) - // this.materialCopy.transparent = true - // this.materialCopy.depthTest = false - // this.materialCopy.depthWrite = false - // this.materialCopy.blending = CustomBlending - // this.materialCopy.blendSrc = DstColorFactor - // this.materialCopy.blendDst = ZeroFactor - // this.materialCopy.blendEquation = AddEquation - // this.materialCopy.blendSrcAlpha = DstAlphaFactor - // this.materialCopy.blendDstAlpha = ZeroFactor - // this.materialCopy.blendEquationAlpha = AddEquation + this.materialCopy.transparent = true + this.materialCopy.depthTest = false + this.materialCopy.depthWrite = false + this.materialCopy.blending = CustomBlending + this.materialCopy.blendSrc = DstColorFactor + this.materialCopy.blendDst = ZeroFactor + this.materialCopy.blendEquation = AddEquation + this.materialCopy.blendSrcAlpha = DstAlphaFactor + this.materialCopy.blendDstAlpha = ZeroFactor + this.materialCopy.blendEquationAlpha = AddEquation // this.materialCopy.blending = CustomBlending // this.materialCopy.blendSrc = OneFactor @@ -32,11 +42,15 @@ export class ApplySAOPass extends Pass { // this.materialCopy.blendSrcAlpha = OneFactor // this.materialCopy.blendDstAlpha = OneFactor // this.materialCopy.blendEquationAlpha = AddEquation - this.materialCopy.uniforms['tDiffuse'].value = srcSao this.materialCopy.needsUpdate = true this.fsQuad = new FullScreenQuad(this.materialCopy) } + public setAoTexture(texture: Texture) { + this.materialCopy.uniforms['tDiffuse'].value = texture + this.materialCopy.needsUpdate = true + } + render(renderer, writeBuffer, readBuffer /*, deltaTime, maskActive*/) { writeBuffer readBuffer diff --git a/packages/viewer/src/modules/pipeline/Pipeline.ts b/packages/viewer/src/modules/pipeline/Pipeline.ts index 9209f67d8..3eae8e508 100644 --- a/packages/viewer/src/modules/pipeline/Pipeline.ts +++ b/packages/viewer/src/modules/pipeline/Pipeline.ts @@ -7,6 +7,11 @@ import { ApplySAOPass } from './ApplySAOPass' import { NormalsType, SpeckleDynamicSAOPass } from './SpeckleDynamicSAOPass' import { SpeckleStaticAOGeneratePass } from './SpeckleStaticAOGeneratePass' +enum RenderType { + NORMAL, + ACCUMULATION +} + export interface PipelineOptions { saoEnabled?: boolean saoParams?: Partial @@ -41,7 +46,9 @@ export class Pipeline { private applySaoPass: ApplySAOPass = null private staticAOGenerationPass: SpeckleStaticAOGeneratePass = null private drawingSize: Vector2 = new Vector2() - public needsRender = true + private _renderType: RenderType = RenderType.NORMAL + private accumulationFrame = 0 + private readonly NUM_ACCUMULATION_FRAMES = 16 public set pipelineOptions(options: PipelineOptions) { Object.assign(this._pipelineOptions, options) @@ -53,6 +60,10 @@ export class Pipeline { } } + private set renderType(value: RenderType) { + this._renderType = value + } + public constructor(renderer: WebGLRenderer, batcher: Batcher) { this._renderer = renderer this._batcher = batcher @@ -74,12 +85,11 @@ export class Pipeline { this.composer.addPass(this.saoPass) this.renderPass = new RenderPass(scene, camera) this.renderPass.renderToScreen = true - this.renderPass.enabled = false + // this.renderPass.enabled = false this.composer.addPass(this.renderPass) this.composer.addPass(this.staticAOGenerationPass) - this.applySaoPass = new ApplySAOPass( - this.staticAOGenerationPass.outputTexture.texture - ) + this.applySaoPass = new ApplySAOPass() + this.applySaoPass.setAoTexture(this.saoPass.saoRenderTarget.texture) this.applySaoPass.renderToScreen = true this.composer.addPass(this.applySaoPass) } @@ -89,17 +99,32 @@ export class Pipeline { this.saoPass.normalMaterial.clippingPlanes = planes } - public render(scene: Scene, camera: Camera) { + public render(scene: Scene, camera: Camera): boolean { this._renderer.getDrawingBufferSize(this.drawingSize) if (this.drawingSize.length() === 0) return - this._renderer.clear(true) - this.renderPass.scene = scene - this.renderPass.camera = camera - this.saoPass.scene = scene - this.saoPass.camera = camera - this.staticAOGenerationPass.update(camera) - this.composer.render() + if (this._renderType === RenderType.NORMAL) { + this._renderer.clear(true) + this.applySaoPass.setAoTexture(this.saoPass.saoRenderTarget.texture) + this.renderPass.scene = scene + this.renderPass.camera = camera + this.saoPass.scene = scene + this.saoPass.camera = camera + this.composer.render() + return true + } else { + this._renderer.clear(true) + this.applySaoPass.setAoTexture(this.staticAOGenerationPass.outputTexture.texture) + this.renderPass.scene = scene + this.renderPass.camera = camera + this.saoPass.scene = scene + this.saoPass.camera = camera + this.staticAOGenerationPass.update(camera, this.accumulationFrame) + this.composer.render() + this.accumulationFrame++ + console.warn('rendering stationary frame => ', this.accumulationFrame) + return this.accumulationFrame < this.NUM_ACCUMULATION_FRAMES ? true : false + } } public resize(width: number, height: number) { @@ -107,10 +132,15 @@ export class Pipeline { } public onStationaryBegin() { - this.needsRender = true + this.renderType = RenderType.ACCUMULATION + this.staticAOGenerationPass.enabled = true + this.accumulationFrame = 0 + console.warn('Starting stationary') } public onStationaryEnd() { - this.needsRender = true + this.renderType = RenderType.NORMAL + this.staticAOGenerationPass.enabled = false + console.warn('Ending stationary') } } diff --git a/packages/viewer/src/modules/pipeline/SpeckleStaticAOGeneratePass.ts b/packages/viewer/src/modules/pipeline/SpeckleStaticAOGeneratePass.ts index 6cd2569cc..2a15ab5ec 100644 --- a/packages/viewer/src/modules/pipeline/SpeckleStaticAOGeneratePass.ts +++ b/packages/viewer/src/modules/pipeline/SpeckleStaticAOGeneratePass.ts @@ -1,10 +1,14 @@ import { + AddEquation, Camera, Color, + CustomBlending, Matrix4, NoBlending, + OneFactor, OrthographicCamera, PerspectiveCamera, + ReverseSubtractEquation, ShaderMaterial, Texture, Vector2, @@ -31,6 +35,7 @@ export class SpeckleStaticAOGeneratePass extends Pass { private _generationBuffer: WebGLRenderTarget private _accumulationBuffer: WebGLRenderTarget private fsQuad: FullScreenQuad + private frameIndex = 0 public set depthTexture(value: Texture) { this._depthTexture = value @@ -75,7 +80,7 @@ export class SpeckleStaticAOGeneratePass extends Pass { bias: { value: 0 }, minResolution: { value: 0.0 }, - kernelRadius: { value: 10.0 }, + kernelRadius: { value: 15.0 }, randomSeed: { value: 0.0 }, frameIndex: { value: 0 } @@ -96,12 +101,18 @@ export class SpeckleStaticAOGeneratePass extends Pass { } }) this.accumulateMaterial.uniforms['tDiffuse'].value = this._generationBuffer.texture - this.accumulateMaterial.blending = NoBlending + this.accumulateMaterial.blending = CustomBlending + this.accumulateMaterial.blendSrc = OneFactor + this.accumulateMaterial.blendDst = OneFactor + this.accumulateMaterial.blendEquation = ReverseSubtractEquation + this.accumulateMaterial.blendSrcAlpha = OneFactor + this.accumulateMaterial.blendDstAlpha = OneFactor + this.accumulateMaterial.blendEquationAlpha = AddEquation this.fsQuad = new FullScreenQuad(this.aoMaterial) } - public update(camera: Camera) { + public update(camera: Camera, frameIndex: number) { this.aoMaterial.uniforms['cameraNear'].value = ( camera as PerspectiveCamera | OrthographicCamera ).near @@ -121,7 +132,9 @@ export class SpeckleStaticAOGeneratePass extends Pass { .isPerspectiveCamera ? 1 : 0 + this.aoMaterial.uniforms['frameIndex'].value = frameIndex this.aoMaterial.needsUpdate = true + this.frameIndex = frameIndex } public render(renderer, writeBuffer, readBuffer) { @@ -134,16 +147,14 @@ export class SpeckleStaticAOGeneratePass extends Pass { const originalClearAlpha = renderer.getClearAlpha() const originalAutoClear = renderer.autoClear - this.renderFrame(renderer, 0) + this.renderFrame(renderer) // restore original state renderer.autoClear = originalAutoClear renderer.setClearColor(originalClearColor) renderer.setClearAlpha(originalClearAlpha) } - private renderFrame(renderer: WebGLRenderer, frameIndex: number) { - this.aoMaterial.uniforms['frameIndex'].value = frameIndex - + private renderFrame(renderer: WebGLRenderer) { renderer.setRenderTarget(this._generationBuffer) renderer.autoClear = false renderer.setClearColor(0x000000) @@ -153,9 +164,11 @@ export class SpeckleStaticAOGeneratePass extends Pass { this.fsQuad.render(renderer) renderer.setRenderTarget(this._accumulationBuffer) - renderer.setClearColor(0x000000) - renderer.setClearAlpha(1) - renderer.clear() + if (this.frameIndex === 0) { + renderer.setClearColor(0xffffff) + renderer.setClearAlpha(1) + renderer.clear() + } this.fsQuad.material = this.accumulateMaterial this.fsQuad.render(renderer) } From 20ddbea6f85985dfe44c5fa9dfb8c7bddbcc0198 Mon Sep 17 00:00:00 2001 From: AlexandruPopovici Date: Wed, 12 Oct 2022 13:05:02 +0300 Subject: [PATCH 05/54] First draft of progressive AO with multiple selectable estimators. WIP on final implementation --- packages/viewer-sandbox/src/Sandbox.ts | 44 ++++++++- packages/viewer-sandbox/src/main.ts | 4 +- .../speckle-static-ao-generate-frag.ts | 85 +++++++++++++++-- .../viewer/src/modules/pipeline/Pipeline.ts | 52 ++++++++++- .../pipeline/SpeckleStaticAOGeneratePass.ts | 93 ++++++++++++++++++- 5 files changed, 261 insertions(+), 17 deletions(-) diff --git a/packages/viewer-sandbox/src/Sandbox.ts b/packages/viewer-sandbox/src/Sandbox.ts index 4d84de649..3a8eeb973 100644 --- a/packages/viewer-sandbox/src/Sandbox.ts +++ b/packages/viewer-sandbox/src/Sandbox.ts @@ -36,7 +36,7 @@ export default class Sandbox { saoBias: 0.15, saoIntensity: 1.25, saoScale: 434, - saoKernelRadius: 10, + saoKernelRadius: 15, saoMinResolution: 0, saoBlur: true, saoBlurRadius: 4, @@ -44,7 +44,12 @@ export default class Sandbox { saoBlurDepthCutoff: 0.0007 }, saoScaleOffset: 0, - saoNormalsRendering: 2 + saoNormalsRendering: 2, + minDistance: 0.0, + maxDistance: 0.008, + ssaoKernelRadius: 0.5, + progressiveAO: 0, + progressive: true } public static lightParams: SunLightConfiguration = { @@ -429,6 +434,41 @@ export default class Sandbox { this.viewer.requestRender() }) + postFolder + .addInput(Sandbox.postParams, 'minDistance', { min: 0, max: 100, step: 0.000001 }) + .on('change', () => { + this.viewer.getRenderer().pipelineOptions = Sandbox.postParams + this.viewer.requestRender() + }) + + postFolder + .addInput(Sandbox.postParams, 'maxDistance', { min: 0, max: 100, step: 0.000001 }) + .on('change', () => { + this.viewer.getRenderer().pipelineOptions = Sandbox.postParams + this.viewer.requestRender() + }) + postFolder + .addInput(Sandbox.postParams, 'ssaoKernelRadius', { min: 0, max: 100 }) + .on('change', () => { + this.viewer.getRenderer().pipelineOptions = Sandbox.postParams + this.viewer.requestRender() + }) + postFolder.addInput(Sandbox.postParams, 'progressive', {}).on('change', () => { + this.viewer.getRenderer().pipelineOptions = Sandbox.postParams + this.viewer.requestRender() + }) + postFolder + .addInput(Sandbox.postParams, 'progressiveAO', { + options: { + SAO: 0, + SSAO: 1 + } + }) + .on('change', () => { + this.viewer.getRenderer().pipelineOptions = Sandbox.postParams + this.viewer.requestRender() + }) + // postFolder // .addInput(Sandbox.postParams.saoParams, 'saoBlurStdDev', { // min: 0, diff --git a/packages/viewer-sandbox/src/main.ts b/packages/viewer-sandbox/src/main.ts index 09401f81a..067770572 100644 --- a/packages/viewer-sandbox/src/main.ts +++ b/packages/viewer-sandbox/src/main.ts @@ -82,7 +82,7 @@ sandbox.makeFilteringUI() await sandbox.loadUrl( // 'https://speckle.xyz/streams/da9e320dad/commits/5388ef24b8?c=%5B-7.66134,10.82932,6.41935,-0.07739,-13.88552,1.8697,0,1%5D' // Revit sample house (good for bim-like stuff with many display meshes) - // 'https://speckle.xyz/streams/da9e320dad/commits/5388ef24b8' + 'https://speckle.xyz/streams/da9e320dad/commits/5388ef24b8' // 'Super' heavy revit shit // 'https://speckle.xyz/streams/e6f9156405/commits/0694d53bb5' // Same sample revit house, local to dim's computer @@ -91,7 +91,7 @@ await sandbox.loadUrl( // IFC building (good for a tree based structure) // 'https://latest.speckle.dev/streams/92b620fb17/commits/2ebd336223' // IFC story, a subtree of the above - 'https://latest.speckle.dev/streams/92b620fb17/objects/8247bbc53865b0e0cb5ee4e252e66216' + // 'https://latest.speckle.dev/streams/92b620fb17/objects/8247bbc53865b0e0cb5ee4e252e66216' // Small scale lines // 'https://speckle.xyz/streams/638d3b1f83/commits/6025e2b546?c=%5B2.18058,-0.20814,9.67642,3.85491,5.05364,0,0,1%5D' // 'https://latest.speckle.dev/streams/3ed8357f29/commits/d10f2af1ce' diff --git a/packages/viewer/src/modules/materials/shaders/speckle-static-ao-generate-frag.ts b/packages/viewer/src/modules/materials/shaders/speckle-static-ao-generate-frag.ts index ece3651aa..8bfdd11ac 100644 --- a/packages/viewer/src/modules/materials/shaders/speckle-static-ao-generate-frag.ts +++ b/packages/viewer/src/modules/materials/shaders/speckle-static-ao-generate-frag.ts @@ -17,6 +17,13 @@ export const speckleStaticAoGenerateFrag = /* glsl */ ` uniform float minResolution; uniform float frameIndex; + #define KERNEL_SIZE 16 + uniform sampler2D tNoise; + uniform vec3 kernel[ KERNEL_SIZE ]; + uniform float minDistance; + uniform float maxDistance; + uniform float ssaoKernelRadius; + #define NUM_SAMPLES 16 #define SPIRAL_TURNS 2 #define NUM_FRAMES 16 @@ -24,8 +31,6 @@ export const speckleStaticAoGenerateFrag = /* glsl */ ` #define NORMAL_TEXTURE 0 #define IMPROVED_NORMAL_RECONSTRUCTION 0 #define ACCURATE_NORMAL_RECONSTRUCTION 1 - - #define AO_ESTIMATOR 0 // RGBA depth #include @@ -37,6 +42,16 @@ export const speckleStaticAoGenerateFrag = /* glsl */ ` return unpackRGBAToDepth( texture2D( tDepth, screenPosition ) ); } + float getLinearDepth( const in vec2 screenPosition ) { + #if PERSPECTIVE_CAMERA == 1 + float fragCoordZ = getDepth(screenPosition); + float viewZ = perspectiveDepthToViewZ( fragCoordZ, cameraNear, cameraFar ); + return viewZToOrthographicDepth( viewZ, cameraNear, cameraFar ); + #else + return texture2D( tDepth, screenPosition ).x; + #endif + } + float getViewZ( const in float depth ) { #if PERSPECTIVE_CAMERA == 1 return perspectiveDepthToViewZ( depth, cameraNear, cameraFar ); @@ -194,16 +209,70 @@ export const speckleStaticAoGenerateFrag = /* glsl */ ` } float sampleViewZ = getViewZ( sampleDepth ); vec3 sampleViewPosition = getViewPosition( sampleUv, sampleDepth, sampleViewZ ); - vec3 viewDelta = sampleViewPosition - centerViewPosition; - float viewDistance = length( viewDelta ); - float scaledScreenDistance = scaleDividedByCameraFar * viewDistance; - occlusionSum += max(0.0, (dot(centerViewNormal, viewDelta) - minResolutionMultipliedByCameraFar) / scaledScreenDistance - bias) / (1.0 + pow2( scaledScreenDistance ) ); + + vec3 v = sampleViewPosition - centerViewPosition; + + float vv = dot(v, v); + float vn = dot(v, centerViewNormal);// - uBias; + + // #if VARIATION == 0 + + // (from the HPG12 paper) + // Note large epsilon to avoid overdarkening within cracks + float radius2 = 2.;//uSampleRadiusWS * uSampleRadiusWS + float epsilon = 0.01; + // occlusionSum += float(vv < radius2) * max(vn / (epsilon + vv), 0.0) / 4.; + + // #elif VARIATION == 1 // default / recommended + + // Smoother transition to zero (lowers contrast, smoothing out corners). [Recommended] + float f = max(radius2 - vv, 0.0) / radius2; + occlusionSum += f * f * f * max(vn / (epsilon + vv), 0.0) / 4.; + + // #elif VARIATION == 2 + + // // Medium contrast (which looks better at high radii), no division. Note that the + // // contribution still falls off with radius^2, but we've adjusted the rate in a way that is + // // more computationally efficient and happens to be aesthetically pleasing. + // float invRadius2 = 1.0 / radius2; + // return 4.0 * max(1.0 - vv * invRadius2, 0.0) * max(vn, 0.0); + + + + // vec3 viewDelta = sampleViewPosition - centerViewPosition; + // float viewDistance = length( viewDelta ); + // float scaledScreenDistance = scaleDividedByCameraFar * viewDistance; + // occlusionSum += max(0.0, (dot(centerViewNormal, viewDelta) - minResolutionMultipliedByCameraFar) / scaledScreenDistance - bias) / (1.0 + pow2( scaledScreenDistance ) ); weightSum += 1.0; } if( weightSum == 0.0 ) discard; return occlusionSum * ( intensity / weightSum ); - #endif - } + #elif AO_ESTIMATOR == 1 + vec3 viewPosition = centerViewPosition; + vec3 viewNormal = getViewNormal( centerViewPosition, vUv, centerDepth ); + vec2 noiseScale = vec2( size.x / 4.0, size.y / 4.0 ); + vec3 random = vec3( texture2D( tNoise, vUv * noiseScale ).r ); + // compute matrix used to reorient a kernel vector + vec3 tangent = normalize( random - viewNormal * dot( random, viewNormal ) ); + vec3 bitangent = cross( viewNormal, tangent ); + mat3 kernelMatrix = mat3( tangent, bitangent, viewNormal ); + float occlusion = 0.0; + for ( int i = 0; i < KERNEL_SIZE; i ++ ) { + vec3 sampleVector = kernelMatrix * kernel[ i ]; // reorient sample vector in view space + vec3 samplePoint = viewPosition + ( sampleVector * ssaoKernelRadius ); // calculate sample point + vec4 samplePointNDC = cameraProjectionMatrix * vec4( samplePoint, 1.0 ); // project point and calculate NDC + samplePointNDC /= samplePointNDC.w; + vec2 samplePointUv = samplePointNDC.xy * 0.5 + 0.5; // compute uv coordinates + float realDepth = getLinearDepth( samplePointUv ); // get linear depth from depth texture + float sampleDepth = viewZToOrthographicDepth( samplePoint.z, cameraNear, cameraFar ); // compute linear depth of the sample view Z value + float delta = sampleDepth - realDepth; + if ( delta > minDistance && delta < maxDistance ) { // if fragment is before sample point, increase occlusion + occlusion += 1.0; + } + } + return clamp( occlusion / float( KERNEL_SIZE ), 0.0, 1.0 ); + #endif + } void main() { float centerDepth = getDepth( vUv ); if( centerDepth >= ( 1.0 - EPSILON ) ) { diff --git a/packages/viewer/src/modules/pipeline/Pipeline.ts b/packages/viewer/src/modules/pipeline/Pipeline.ts index 3eae8e508..f8070d5c4 100644 --- a/packages/viewer/src/modules/pipeline/Pipeline.ts +++ b/packages/viewer/src/modules/pipeline/Pipeline.ts @@ -17,6 +17,11 @@ export interface PipelineOptions { saoParams?: Partial saoScaleOffset?: number saoNormalsRendering?: NormalsType + minDistance?: number + maxDistance?: number + ssaoKernelRadius?: number + progressiveAO?: number + progressive?: boolean } export const DefaultPipelineOptions: PipelineOptions = { @@ -33,7 +38,12 @@ export const DefaultPipelineOptions: PipelineOptions = { saoBlurDepthCutoff: 0.0007 }, saoScaleOffset: 0, - saoNormalsRendering: NormalsType.ACCURATE + saoNormalsRendering: NormalsType.ACCURATE, + minDistance: 0, + maxDistance: 0.008, + ssaoKernelRadius: 0.5, + progressiveAO: 0, + progressive: true } export class Pipeline { @@ -49,6 +59,7 @@ export class Pipeline { private _renderType: RenderType = RenderType.NORMAL private accumulationFrame = 0 private readonly NUM_ACCUMULATION_FRAMES = 16 + private enableProgressive = true public set pipelineOptions(options: PipelineOptions) { Object.assign(this._pipelineOptions, options) @@ -57,6 +68,43 @@ export class Pipeline { Object.assign(this.saoPass.params, this._pipelineOptions.saoParams) this.saoPass.params.saoScale += this._pipelineOptions.saoScaleOffset this.saoPass.normalsRendering = this._pipelineOptions.saoNormalsRendering + if ( + this.staticAOGenerationPass.minDistance !== this._pipelineOptions.minDistance || + this.staticAOGenerationPass.maxDistance !== this._pipelineOptions.maxDistance + ) + this.accumulationFrame = 0 + if ( + this._pipelineOptions.ssaoKernelRadius !== undefined && + this.staticAOGenerationPass.ssaoKernelRadius !== + this._pipelineOptions.ssaoKernelRadius + ) { + this.accumulationFrame = 0 + this.staticAOGenerationPass.ssaoKernelRadius = + this._pipelineOptions.ssaoKernelRadius + } + if ( + this._pipelineOptions.progressiveAO !== undefined && + this.staticAOGenerationPass.progressiveAO !== + this._pipelineOptions.progressiveAO + ) { + this.accumulationFrame = 0 + this.staticAOGenerationPass.progressiveAO = this._pipelineOptions.progressiveAO + this.staticAOGenerationPass.aoMaterial.defines['AO_ESTIMATOR'] = + this._pipelineOptions.progressiveAO + this.staticAOGenerationPass.aoMaterial.needsUpdate = true + } + this.staticAOGenerationPass.minDistance = this._pipelineOptions.minDistance + this.staticAOGenerationPass.maxDistance = this._pipelineOptions.maxDistance + if ( + this._pipelineOptions.saoParams.saoKernelRadius !== undefined && + this.staticAOGenerationPass.kernelRadius !== + this._pipelineOptions.saoParams.saoKernelRadius + ) { + this.accumulationFrame = 0 + this.staticAOGenerationPass.kernelRadius = + this._pipelineOptions.saoParams.saoKernelRadius + } + this.enableProgressive = this._pipelineOptions.progressive } } @@ -112,7 +160,7 @@ export class Pipeline { this.saoPass.camera = camera this.composer.render() return true - } else { + } else if (this.enableProgressive) { this._renderer.clear(true) this.applySaoPass.setAoTexture(this.staticAOGenerationPass.outputTexture.texture) this.renderPass.scene = scene diff --git a/packages/viewer/src/modules/pipeline/SpeckleStaticAOGeneratePass.ts b/packages/viewer/src/modules/pipeline/SpeckleStaticAOGeneratePass.ts index 2a15ab5ec..3d98cacd3 100644 --- a/packages/viewer/src/modules/pipeline/SpeckleStaticAOGeneratePass.ts +++ b/packages/viewer/src/modules/pipeline/SpeckleStaticAOGeneratePass.ts @@ -3,15 +3,21 @@ import { Camera, Color, CustomBlending, + DataTexture, + FloatType, + MathUtils, Matrix4, NoBlending, OneFactor, OrthographicCamera, PerspectiveCamera, + RedFormat, + RepeatWrapping, ReverseSubtractEquation, ShaderMaterial, Texture, Vector2, + Vector3, WebGLRenderer, WebGLRenderTarget } from 'three' @@ -22,13 +28,14 @@ import { speckleStaticAoGenerateVert } from '../materials/shaders/speckle-static import { speckleStaticAoGenerateFrag } from '../materials/shaders/speckle-static-ao-generate-frag' import { speckleStaticAoAccumulateVert } from '../materials/shaders/speckle-static-ao-accumulate-vert' import { speckleStaticAoAccumulateFrag } from '../materials/shaders/speckle-static-ao-accumulate-frag' +import { SimplexNoise } from 'three/examples/jsm//math/SimplexNoise.js' /** * SAO implementation inspired from bhouston previous SAO work */ export class SpeckleStaticAOGeneratePass extends Pass { private batcher: Batcher = null - private aoMaterial: ShaderMaterial = null + public aoMaterial: ShaderMaterial = null private accumulateMaterial: ShaderMaterial = null private _depthTexture: Texture private _normalTexture: Texture @@ -36,6 +43,14 @@ export class SpeckleStaticAOGeneratePass extends Pass { private _accumulationBuffer: WebGLRenderTarget private fsQuad: FullScreenQuad private frameIndex = 0 + private kernels: Array> = new Array(16) + private kernelSize = 16 + private noiseTextures: Array = [] + public minDistance = 0.02 + public maxDistance = 0.3 + public ssaoKernelRadius = 1 + public progressiveAO = 0 + public kernelRadius = 15 public set depthTexture(value: Texture) { this._depthTexture = value @@ -83,7 +98,13 @@ export class SpeckleStaticAOGeneratePass extends Pass { kernelRadius: { value: 15.0 }, randomSeed: { value: 0.0 }, - frameIndex: { value: 0 } + frameIndex: { value: 0 }, + + tNoise: { value: null }, + kernel: { value: null }, + minDistance: { value: 0.0 }, + maxDistance: { value: 0.008 }, + ssaoKernelRadius: { value: 0.5 } } }) @@ -132,9 +153,22 @@ export class SpeckleStaticAOGeneratePass extends Pass { .isPerspectiveCamera ? 1 : 0 + this.aoMaterial.uniforms['kernelRadius'].value = this.kernelRadius this.aoMaterial.uniforms['frameIndex'].value = frameIndex - this.aoMaterial.needsUpdate = true this.frameIndex = frameIndex + + if (!this.kernels[frameIndex]) { + this.generateSampleKernel(frameIndex) + } + if (!this.noiseTextures[frameIndex]) { + this.generateRandomKernelRotations(frameIndex) + } + this.aoMaterial.uniforms['kernel'].value = this.kernels[frameIndex] + this.aoMaterial.uniforms['tNoise'].value = this.noiseTextures[frameIndex] + this.aoMaterial.uniforms['ssaoKernelRadius'].value = this.ssaoKernelRadius + this.aoMaterial.uniforms['minDistance'].value = this.minDistance + this.aoMaterial.uniforms['maxDistance'].value = this.maxDistance + this.aoMaterial.needsUpdate = true } public render(renderer, writeBuffer, readBuffer) { @@ -180,4 +214,57 @@ export class SpeckleStaticAOGeneratePass extends Pass { this.aoMaterial.uniforms['size'].value.set(width, height) this.aoMaterial.needsUpdate = true } + + private generateSampleKernel(frameIndex: number) { + const kernelSize = this.kernelSize + this.kernels[frameIndex] = [] + + for (let i = 0; i < kernelSize; i++) { + const sample = new Vector3() + sample.x = Math.random() * 2 - 1 + sample.y = Math.random() * 2 - 1 + sample.z = Math.random() + + sample.normalize() + + let scale = i / kernelSize + scale = MathUtils.lerp(0.1, 1, scale * scale) + sample.multiplyScalar(scale) + + this.kernels[frameIndex].push(sample) + } + } + + private generateRandomKernelRotations(frameIndex: number) { + const width = 4, + height = 4 + + if (SimplexNoise === undefined) { + console.error('THREE.SSAOPass: The pass relies on SimplexNoise.') + } + + const simplex = new SimplexNoise() + + const size = width * height + const data = new Float32Array(size) + + for (let i = 0; i < size; i++) { + const x = Math.random() * 2 - 1 + const y = Math.random() * 2 - 1 + const z = 0 + + data[i] = simplex.noise3d(x, y, z) + } + + this.noiseTextures[frameIndex] = new DataTexture( + data, + width, + height, + RedFormat, + FloatType + ) + this.noiseTextures[frameIndex].wrapS = RepeatWrapping + this.noiseTextures[frameIndex].wrapT = RepeatWrapping + this.noiseTextures[frameIndex].needsUpdate = true + } } From 75a2a6ca93071fee67008fb29c842b265cdae6da Mon Sep 17 00:00:00 2001 From: AlexandruPopovici Date: Wed, 12 Oct 2022 18:59:07 +0300 Subject: [PATCH 06/54] Restructuring how the pipeline and it's passes work so we can extend/configure them more easily. Added the concept of SpecklePass --- .../viewer/src/modules/pipeline/DepthPass.ts | 91 ++++ .../viewer/src/modules/pipeline/Pipeline.ts | 127 ++--- .../modules/pipeline/SpeckleDynamicSAOPass.ts | 448 +++++++++--------- .../pipeline/SpeckleStaticAOGeneratePass.ts | 1 - 4 files changed, 346 insertions(+), 321 deletions(-) create mode 100644 packages/viewer/src/modules/pipeline/DepthPass.ts diff --git a/packages/viewer/src/modules/pipeline/DepthPass.ts b/packages/viewer/src/modules/pipeline/DepthPass.ts new file mode 100644 index 000000000..6b1397f81 --- /dev/null +++ b/packages/viewer/src/modules/pipeline/DepthPass.ts @@ -0,0 +1,91 @@ +import { + Camera, + Color, + DoubleSide, + NoBlending, + RGBADepthPacking, + Scene, + Texture, + WebGLRenderTarget +} from 'three' +import { Pass } from 'three/examples/jsm/postprocessing/Pass' +import SpeckleDepthMaterial from '../materials/SpeckleDepthMaterial' +import { SpecklePass } from './Pipeline' + +export class DepthPass extends Pass implements SpecklePass { + private renderTarget: WebGLRenderTarget + private depthMaterial: SpeckleDepthMaterial = null + private scene: Scene + private camera: Camera + + private colorBuffer: Color = new Color() + + get displayName(): string { + return 'DEPTH' + } + + get outputTexture(): Texture { + return this.renderTarget.texture + } + + constructor() { + super() + + this.renderTarget = new WebGLRenderTarget(256, 256) + /** On Chromium, on MacOS the 16 bit depth render buffer appears broken. + * We're not really using a stencil buffer at all, we're just forcing + * three.js to use a 24 bit depth render buffer + */ + this.renderTarget.depthBuffer = true + this.renderTarget.stencilBuffer = true + + this.depthMaterial = new SpeckleDepthMaterial( + { + depthPacking: RGBADepthPacking + }, + ['USE_RTE', 'ALPHATEST_REJECTION'] + ) + this.depthMaterial.blending = NoBlending + this.depthMaterial.side = DoubleSide + } + + public update(camera: Camera, scene: Scene) { + this.camera = camera + this.scene = scene + } + + public render(renderer, writeBuffer, readBuffer) { + writeBuffer + readBuffer + + renderer.getClearColor(this.colorBuffer) + const originalClearAlpha = renderer.getClearAlpha() + const originalAutoClear = renderer.autoClear + + renderer.setRenderTarget(this.renderTarget) + renderer.autoClear = false + + renderer.setClearColor(0x000000) + renderer.setClearAlpha(1.0) + renderer.clear() + + const shadowmapEnabled = renderer.shadowMap.enabled + const shadowmapNeedsUpdate = renderer.shadowMap.needsUpdate + this.scene.overrideMaterial = this.depthMaterial + renderer.shadowMap.enabled = false + renderer.shadowMap.needsUpdate = false + renderer.render(this.scene, this.camera) + renderer.shadowMap.enabled = shadowmapEnabled + renderer.shadowMap.needsUpdate = shadowmapNeedsUpdate + this.scene.overrideMaterial = null + + // restore original state + renderer.autoClear = originalAutoClear + renderer.setClearColor(this.colorBuffer) + renderer.setClearAlpha(originalClearAlpha) + } + + public setSize(width: number, height: number) { + this.renderTarget.setSize(width, height) + } +} diff --git a/packages/viewer/src/modules/pipeline/Pipeline.ts b/packages/viewer/src/modules/pipeline/Pipeline.ts index f8070d5c4..86d5ff19c 100644 --- a/packages/viewer/src/modules/pipeline/Pipeline.ts +++ b/packages/viewer/src/modules/pipeline/Pipeline.ts @@ -1,111 +1,61 @@ -import { Camera, Plane, Scene, Vector2, WebGLRenderer } from 'three' +import { Camera, Plane, Scene, Texture, Vector2, WebGLRenderer } from 'three' import { EffectComposer } from 'three/examples/jsm/postprocessing/EffectComposer.js' import { RenderPass } from 'three/examples/jsm/postprocessing/RenderPass.js' -import { SAOPassParams } from 'three/examples/jsm/postprocessing/SAOPass.js' import Batcher from '../batching/Batcher' import { ApplySAOPass } from './ApplySAOPass' -import { NormalsType, SpeckleDynamicSAOPass } from './SpeckleDynamicSAOPass' +import { + DefaultSpeckleDynamicSAOPassParams, + NormalsType, + SpeckleDynamicSAOPass, + SpeckleDynamicSAOPassParams +} from './SpeckleDynamicSAOPass' import { SpeckleStaticAOGeneratePass } from './SpeckleStaticAOGeneratePass' enum RenderType { NORMAL, ACCUMULATION } +export interface SpecklePass { + get displayName(): string + get outputTexture(): Texture +} export interface PipelineOptions { - saoEnabled?: boolean - saoParams?: Partial - saoScaleOffset?: number - saoNormalsRendering?: NormalsType - minDistance?: number - maxDistance?: number - ssaoKernelRadius?: number - progressiveAO?: number - progressive?: boolean + dynamicAoEnabled: boolean + dynamicAoParams: SpeckleDynamicSAOPassParams } export const DefaultPipelineOptions: PipelineOptions = { - saoEnabled: true, - saoParams: { - saoBias: 0.15, - saoIntensity: 1.25, - saoScale: 434, - saoKernelRadius: 10, - saoMinResolution: 0, - saoBlur: true, - saoBlurRadius: 4, - saoBlurStdDev: 4, - saoBlurDepthCutoff: 0.0007 - }, - saoScaleOffset: 0, - saoNormalsRendering: NormalsType.ACCURATE, - minDistance: 0, - maxDistance: 0.008, - ssaoKernelRadius: 0.5, - progressiveAO: 0, - progressive: true + dynamicAoEnabled: true, + dynamicAoParams: DefaultSpeckleDynamicSAOPassParams + // saoScaleOffset: 0, + // saoNormalsRendering: NormalsType.ACCURATE, + // minDistance: 0, + // maxDistance: 0.008, + // ssaoKernelRadius: 0.5, + // progressiveAO: 0, + // progressive: true } export class Pipeline { private _renderer: WebGLRenderer = null private _batcher: Batcher = null - private _pipelineOptions: PipelineOptions = {} + private _pipelineOptions: PipelineOptions = Object.assign({}, DefaultPipelineOptions) private composer: EffectComposer = null + private renderPass: RenderPass = null - private saoPass: SpeckleDynamicSAOPass = null + private dynamicAoPass: SpeckleDynamicSAOPass = null private applySaoPass: ApplySAOPass = null private staticAOGenerationPass: SpeckleStaticAOGeneratePass = null + private drawingSize: Vector2 = new Vector2() private _renderType: RenderType = RenderType.NORMAL private accumulationFrame = 0 private readonly NUM_ACCUMULATION_FRAMES = 16 private enableProgressive = true - public set pipelineOptions(options: PipelineOptions) { + public set pipelineOptions(options: Partial) { Object.assign(this._pipelineOptions, options) - if (this.saoPass) { - this.applySaoPass.enabled = this._pipelineOptions.saoEnabled - Object.assign(this.saoPass.params, this._pipelineOptions.saoParams) - this.saoPass.params.saoScale += this._pipelineOptions.saoScaleOffset - this.saoPass.normalsRendering = this._pipelineOptions.saoNormalsRendering - if ( - this.staticAOGenerationPass.minDistance !== this._pipelineOptions.minDistance || - this.staticAOGenerationPass.maxDistance !== this._pipelineOptions.maxDistance - ) - this.accumulationFrame = 0 - if ( - this._pipelineOptions.ssaoKernelRadius !== undefined && - this.staticAOGenerationPass.ssaoKernelRadius !== - this._pipelineOptions.ssaoKernelRadius - ) { - this.accumulationFrame = 0 - this.staticAOGenerationPass.ssaoKernelRadius = - this._pipelineOptions.ssaoKernelRadius - } - if ( - this._pipelineOptions.progressiveAO !== undefined && - this.staticAOGenerationPass.progressiveAO !== - this._pipelineOptions.progressiveAO - ) { - this.accumulationFrame = 0 - this.staticAOGenerationPass.progressiveAO = this._pipelineOptions.progressiveAO - this.staticAOGenerationPass.aoMaterial.defines['AO_ESTIMATOR'] = - this._pipelineOptions.progressiveAO - this.staticAOGenerationPass.aoMaterial.needsUpdate = true - } - this.staticAOGenerationPass.minDistance = this._pipelineOptions.minDistance - this.staticAOGenerationPass.maxDistance = this._pipelineOptions.maxDistance - if ( - this._pipelineOptions.saoParams.saoKernelRadius !== undefined && - this.staticAOGenerationPass.kernelRadius !== - this._pipelineOptions.saoParams.saoKernelRadius - ) { - this.accumulationFrame = 0 - this.staticAOGenerationPass.kernelRadius = - this._pipelineOptions.saoParams.saoKernelRadius - } - this.enableProgressive = this._pipelineOptions.progressive - } } private set renderType(value: RenderType) { @@ -121,7 +71,7 @@ export class Pipeline { } public configure(scene: Scene, camera: Camera) { - this.saoPass = new SpeckleDynamicSAOPass( + this.dynamicAoPass = new SpeckleDynamicSAOPass( scene, camera, this._batcher, @@ -129,22 +79,23 @@ export class Pipeline { NormalsType.IMPROVED ) this.staticAOGenerationPass = new SpeckleStaticAOGeneratePass(this._batcher) - this.staticAOGenerationPass.depthTexture = this.saoPass.depthRenderTarget.texture - this.composer.addPass(this.saoPass) + this.staticAOGenerationPass.depthTexture = + this.dynamicAoPass.depthRenderTarget.texture + this.composer.addPass(this.dynamicAoPass) this.renderPass = new RenderPass(scene, camera) this.renderPass.renderToScreen = true // this.renderPass.enabled = false this.composer.addPass(this.renderPass) this.composer.addPass(this.staticAOGenerationPass) this.applySaoPass = new ApplySAOPass() - this.applySaoPass.setAoTexture(this.saoPass.saoRenderTarget.texture) + this.applySaoPass.setAoTexture(this.dynamicAoPass.saoRenderTarget.texture) this.applySaoPass.renderToScreen = true this.composer.addPass(this.applySaoPass) } public updateClippingPlanes(planes: Plane[]) { - this.saoPass.depthMaterial.clippingPlanes = planes - this.saoPass.normalMaterial.clippingPlanes = planes + this.dynamicAoPass.depthMaterial.clippingPlanes = planes + this.dynamicAoPass.normalMaterial.clippingPlanes = planes } public render(scene: Scene, camera: Camera): boolean { @@ -153,11 +104,11 @@ export class Pipeline { if (this._renderType === RenderType.NORMAL) { this._renderer.clear(true) - this.applySaoPass.setAoTexture(this.saoPass.saoRenderTarget.texture) + this.applySaoPass.setAoTexture(this.dynamicAoPass.saoRenderTarget.texture) this.renderPass.scene = scene this.renderPass.camera = camera - this.saoPass.scene = scene - this.saoPass.camera = camera + this.dynamicAoPass.scene = scene + this.dynamicAoPass.camera = camera this.composer.render() return true } else if (this.enableProgressive) { @@ -165,8 +116,8 @@ export class Pipeline { this.applySaoPass.setAoTexture(this.staticAOGenerationPass.outputTexture.texture) this.renderPass.scene = scene this.renderPass.camera = camera - this.saoPass.scene = scene - this.saoPass.camera = camera + this.dynamicAoPass.scene = scene + this.dynamicAoPass.camera = camera this.staticAOGenerationPass.update(camera, this.accumulationFrame) this.composer.render() this.accumulationFrame++ diff --git a/packages/viewer/src/modules/pipeline/SpeckleDynamicSAOPass.ts b/packages/viewer/src/modules/pipeline/SpeckleDynamicSAOPass.ts index b47fe07e7..c29a0c1e1 100644 --- a/packages/viewer/src/modules/pipeline/SpeckleDynamicSAOPass.ts +++ b/packages/viewer/src/modules/pipeline/SpeckleDynamicSAOPass.ts @@ -1,25 +1,23 @@ import { - AdditiveBlending, Camera, - DoubleSide, + Color, NoBlending, OrthographicCamera, PerspectiveCamera, - RGBADepthPacking, Scene, ShaderMaterial, + Texture, UniformsUtils, - Vector2 + Vector2, + WebGLRenderTarget } from 'three' -import { FullScreenQuad } from 'three/examples/jsm/postprocessing/Pass' -import { SAOPass } from 'three/examples/jsm/postprocessing/SAOPass.js' -import { BlurShaderUtils } from 'three/examples/jsm/shaders/DepthLimitedBlurShader.js' +import { FullScreenQuad, Pass } from 'three/examples/jsm/postprocessing/Pass' import { speckleSaoFrag } from '../materials/shaders/speckle-sao-frag' import { speckleSaoVert } from '../materials/shaders/speckle-sao-vert' import { SAOShader } from 'three/examples/jsm/shaders/SAOShader.js' -import Batcher from '../batching/Batcher' -import SpeckleDepthMaterial from '../materials/SpeckleDepthMaterial' -import SpeckleNormalMaterial from '../materials/SpeckleNormalMaterial' +import { DepthLimitedBlurShader } from 'three/examples/jsm/shaders/DepthLimitedBlurShader.js' +import { BlurShaderUtils } from 'three/examples/jsm/shaders/DepthLimitedBlurShader.js' +import { SpecklePass } from './Pipeline' export enum NormalsType { DEFAULT = 0, @@ -27,62 +25,75 @@ export enum NormalsType { ACCURATE = 2 } +export interface SpeckleDynamicSAOPassParams { + intensity: number + scale: number + kernelRadius: number + bias: number + normalsType: NormalsType + blurEnabled: boolean + blurRadius: number + blurStdDev: number + blurDepthCutoff: number +} + +export const DefaultSpeckleDynamicSAOPassParams = { + intensity: 1.25, + scale: 0, + kernelRadius: 10, + bias: 0.15, + normalsType: NormalsType.ACCURATE, + blurEnabled: true, + blurRadius: 4, + blurStdDev: 4, + blurDepthCutoff: 0.0007 +} + /** * SAO implementation inspired from bhouston previous SAO work */ -export class SpeckleDynamicSAOPass extends SAOPass { - private _oldClearColor - private prevStdDev - private prevNumSamples - private batcher: Batcher = null - private normalsType: NormalsType = NormalsType.IMPROVED +export class SpeckleDynamicSAOPass extends Pass implements SpecklePass { + private params: SpeckleDynamicSAOPassParams + private colorBuffer: Color = new Color() + private saoMaterial: ShaderMaterial = null + private vBlurMaterial: ShaderMaterial = null + private hBlurMaterial: ShaderMaterial = null + private saoRenderTarget: WebGLRenderTarget = null + private blurIntermediateRenderTarget: WebGLRenderTarget = null + private fsQuad: FullScreenQuad = null - public set normalsRendering(type: NormalsType) { - this.normalsType = type - this.saoMaterial.defines['NORMAL_TEXTURE'] = - this.normalsType === NormalsType.DEFAULT ? 1 : 0 - this.saoMaterial.defines['IMPROVED_NORMAL_RECONSTRUCTION'] = - this.normalsType === NormalsType.IMPROVED ? 1 : 0 - this.saoMaterial.defines['ACCURATE_NORMAL_RECONSTRUCTION'] = - this.normalsType === NormalsType.ACCURATE ? 1 : 0 - this.saoMaterial.needsUpdate = true + private prevStdDev: number + private prevNumSamples: number + + get displayName(): string { + return 'SAO' } - constructor( - scene: Scene, - camera: Camera, - batcher: Batcher, - useDepthTexture = false, - normalsType: NormalsType, - resolution = new Vector2(256, 256) - ) { - super(scene, camera, useDepthTexture, true, resolution) + get outputTexture(): Texture { + return this.saoRenderTarget.texture + } - this.batcher = batcher + public setDepthTexture(texture: Texture) { + this.saoMaterial.uniforms['tDepth'].value = texture + this.vBlurMaterial.uniforms['tDepth'].value = texture + this.hBlurMaterial.uniforms['tDepth'].value = texture + this.saoMaterial.needsUpdate = true + this.vBlurMaterial.needsUpdate = true + this.hBlurMaterial.needsUpdate = true + } - /** On Chromium, on MacOS the 16 bit depth render buffer appears broken. - * We're not really using a stencil buffer at all, we're just forcing - * three.js to use a 24 bit depth render buffer - */ - this.depthRenderTarget.depthBuffer = true - this.depthRenderTarget.stencilBuffer = true - this.normalRenderTarget.depthBuffer = true - this.normalRenderTarget.stencilBuffer = true + constructor() { + super() - this.depthMaterial = new SpeckleDepthMaterial( - { - depthPacking: RGBADepthPacking - }, - ['USE_RTE', 'ALPHATEST_REJECTION'] - ) - this.depthMaterial.blending = NoBlending - this.depthMaterial.side = DoubleSide - - this.normalMaterial = new SpeckleNormalMaterial({}, ['USE_RTE']) - this.normalMaterial.blending = NoBlending - this.normalMaterial.side = DoubleSide + // this.normalRenderTarget.depthBuffer = true + // this.normalRenderTarget.stencilBuffer = true + // this.normalMaterial = new SpeckleNormalMaterial({}, ['USE_RTE']) + // this.normalMaterial.blending = NoBlending + // this.normalMaterial.side = DoubleSide + this.saoRenderTarget = new WebGLRenderTarget(256, 256) + this.blurIntermediateRenderTarget = new WebGLRenderTarget(256, 256) this.saoMaterial = new ShaderMaterial({ defines: { NUM_SAMPLES: 7, @@ -96,211 +107,184 @@ export class SpeckleDynamicSAOPass extends SAOPass { vertexShader: speckleSaoVert, uniforms: UniformsUtils.clone(SAOShader.uniforms) }) - this.normalsRendering = normalsType this.saoMaterial.extensions.derivatives = true - this.saoMaterial.defines['DEPTH_PACKING'] = this.supportsDepthTextureExtension - ? 0 - : 1 - this.saoMaterial.defines['PERSPECTIVE_CAMERA'] = (this.camera as PerspectiveCamera) + this.saoMaterial.defines['DEPTH_PACKING'] = 1 + this.saoMaterial.uniforms['tDepth'].value = null + this.saoMaterial.uniforms['tNormal'].value = null + this.saoMaterial.uniforms['size'].value.set(256, 256) + this.saoMaterial.uniforms['minResolution'].value = 0 + this.saoMaterial.blending = NoBlending + + this.vBlurMaterial = new ShaderMaterial({ + uniforms: UniformsUtils.clone(DepthLimitedBlurShader.uniforms), + defines: Object.assign({}, DepthLimitedBlurShader.defines), + vertexShader: DepthLimitedBlurShader.vertexShader, + fragmentShader: DepthLimitedBlurShader.fragmentShader + }) + this.vBlurMaterial.defines['DEPTH_PACKING'] = 1 + + this.vBlurMaterial.uniforms['tDiffuse'].value = this.saoRenderTarget.texture + this.vBlurMaterial.uniforms['tDepth'].value = null + this.vBlurMaterial.uniforms['size'].value.set(256, 256) + this.vBlurMaterial.blending = NoBlending + + this.hBlurMaterial = new ShaderMaterial({ + uniforms: UniformsUtils.clone(DepthLimitedBlurShader.uniforms), + defines: Object.assign({}, DepthLimitedBlurShader.defines), + vertexShader: DepthLimitedBlurShader.vertexShader, + fragmentShader: DepthLimitedBlurShader.fragmentShader + }) + this.hBlurMaterial.defines['DEPTH_PACKING'] = 1 + + this.hBlurMaterial.uniforms['tDiffuse'].value = + this.blurIntermediateRenderTarget.texture + this.hBlurMaterial.uniforms['tDepth'].value = null + this.hBlurMaterial.uniforms['size'].value.set(256, 256) + this.hBlurMaterial.blending = NoBlending + + this.fsQuad = new FullScreenQuad(this.saoMaterial) + } + + public update(scene: Scene, camera: Camera) { + /** SAO DEFINES */ + this.saoMaterial.defines['PERSPECTIVE_CAMERA'] = (camera as PerspectiveCamera) .isPerspectiveCamera ? 1 : 0 - this.saoMaterial.uniforms['tDepth'].value = this.supportsDepthTextureExtension - ? this.beautyRenderTarget.depthTexture - : this.depthRenderTarget.texture - this.saoMaterial.uniforms['tNormal'].value = this.normalRenderTarget.texture - this.saoMaterial.uniforms['size'].value.set(this.resolution.x, this.resolution.y) + + this.saoMaterial.defines['NORMAL_TEXTURE'] = + this.params.normalsType === NormalsType.DEFAULT ? 1 : 0 + this.saoMaterial.defines['IMPROVED_NORMAL_RECONSTRUCTION'] = + this.params.normalsType === NormalsType.IMPROVED ? 1 : 0 + this.saoMaterial.defines['ACCURATE_NORMAL_RECONSTRUCTION'] = + this.params.normalsType === NormalsType.ACCURATE ? 1 : 0 + + /** SAO UNIFORMS */ + this.saoMaterial.uniforms['cameraNear'].value = ( + camera as PerspectiveCamera | OrthographicCamera + ).near + this.saoMaterial.uniforms['cameraFar'].value = ( + camera as PerspectiveCamera | OrthographicCamera + ).far this.saoMaterial.uniforms['cameraInverseProjectionMatrix'].value.copy( - this.camera.projectionMatrixInverse + camera.projectionMatrixInverse ) - this.saoMaterial.uniforms['cameraProjectionMatrix'].value = - this.camera.projectionMatrix - this.saoMaterial.blending = NoBlending + this.saoMaterial.uniforms['cameraProjectionMatrix'].value = camera.projectionMatrix + + /** SAO UNIFORM PARAMS */ + this.saoMaterial.uniforms['intensity'].value = this.params.intensity + this.saoMaterial.uniforms['scale'].value = this.params.scale + this.saoMaterial.uniforms['kernelRadius'].value = this.params.kernelRadius + this.saoMaterial.uniforms['bias'].value = this.params.bias + + this.saoMaterial.needsUpdate = true + + /** BLUR DEFINES */ + this.vBlurMaterial.defines['PERSPECTIVE_CAMERA'] = (camera as PerspectiveCamera) + .isPerspectiveCamera + ? 1 + : 0 + this.hBlurMaterial.defines['PERSPECTIVE_CAMERA'] = (camera as PerspectiveCamera) + .isPerspectiveCamera + ? 1 + : 0 + + /** BLUR UNIFORMS */ + this.vBlurMaterial.uniforms['cameraNear'].value = ( + camera as PerspectiveCamera | OrthographicCamera + ).near + this.vBlurMaterial.uniforms['cameraFar'].value = ( + camera as PerspectiveCamera | OrthographicCamera + ).far + this.hBlurMaterial.uniforms['cameraNear'].value = ( + camera as PerspectiveCamera | OrthographicCamera + ).near + this.hBlurMaterial.uniforms['cameraFar'].value = ( + camera as PerspectiveCamera | OrthographicCamera + ).far + + /** BLUR UNIFORM PARAMS */ + const depthCutoff = + this.params.blurDepthCutoff * + ((camera as PerspectiveCamera | OrthographicCamera).far - + (camera as PerspectiveCamera | OrthographicCamera).near) + this.vBlurMaterial.uniforms['depthCutoff'].value = depthCutoff + this.hBlurMaterial.uniforms['depthCutoff'].value = depthCutoff + + this.params.blurRadius = Math.floor(this.params.blurRadius) + if ( + this.prevStdDev !== this.params.blurStdDev || + this.prevNumSamples !== this.params.blurRadius + ) { + BlurShaderUtils.configure( + this.vBlurMaterial, + this.params.blurRadius, + this.params.blurStdDev, + new Vector2(0, 1) + ) + BlurShaderUtils.configure( + this.hBlurMaterial, + this.params.blurRadius, + this.params.blurStdDev, + new Vector2(1, 0) + ) + this.prevStdDev = this.params.blurStdDev + this.prevNumSamples = this.params.blurRadius + } + this.vBlurMaterial.needsUpdate = true + this.hBlurMaterial.needsUpdate = true } public render(renderer, writeBuffer, readBuffer) { writeBuffer readBuffer - if (this.params.output === 1) { - return - } - renderer.getClearColor(this._oldClearColor) - this.oldClearAlpha = renderer.getClearAlpha() - renderer.autoClear = false - - renderer.setRenderTarget(this.depthRenderTarget) - renderer.clear() - this.saoMaterial.blending = AdditiveBlending - this.saoMaterial.uniforms['bias'].value = this.params.saoBias - this.saoMaterial.uniforms['intensity'].value = this.params.saoIntensity - this.saoMaterial.uniforms['scale'].value = this.params.saoScale - this.saoMaterial.uniforms['kernelRadius'].value = this.params.saoKernelRadius - this.saoMaterial.uniforms['minResolution'].value = this.params.saoMinResolution - this.saoMaterial.uniforms['cameraNear'].value = ( - this.camera as PerspectiveCamera | OrthographicCamera - ).near - this.saoMaterial.uniforms['cameraFar'].value = ( - this.camera as PerspectiveCamera | OrthographicCamera - ).far - this.saoMaterial.uniforms['cameraInverseProjectionMatrix'].value.copy( - this.camera.projectionMatrixInverse - ) - this.saoMaterial.uniforms['cameraProjectionMatrix'].value = - this.camera.projectionMatrix - // this.saoMaterial.uniforms['randomSeed'].value = Math.random() - - const depthCutoff = - this.params.saoBlurDepthCutoff * - ((this.camera as PerspectiveCamera | OrthographicCamera).far - - (this.camera as PerspectiveCamera | OrthographicCamera).near) - this.vBlurMaterial.uniforms['depthCutoff'].value = depthCutoff - this.hBlurMaterial.uniforms['depthCutoff'].value = depthCutoff - - this.vBlurMaterial.uniforms['cameraNear'].value = ( - this.camera as PerspectiveCamera | OrthographicCamera - ).near - this.vBlurMaterial.uniforms['cameraFar'].value = ( - this.camera as PerspectiveCamera | OrthographicCamera - ).far - this.hBlurMaterial.uniforms['cameraNear'].value = ( - this.camera as PerspectiveCamera | OrthographicCamera - ).near - this.hBlurMaterial.uniforms['cameraFar'].value = ( - this.camera as PerspectiveCamera | OrthographicCamera - ).far - - this.params.saoBlurRadius = Math.floor(this.params.saoBlurRadius) - if ( - this.prevStdDev !== this.params.saoBlurStdDev || - this.prevNumSamples !== this.params.saoBlurRadius - ) { - BlurShaderUtils.configure( - this.vBlurMaterial, - this.params.saoBlurRadius, - this.params.saoBlurStdDev, - new Vector2(0, 1) - ) - BlurShaderUtils.configure( - this.hBlurMaterial, - this.params.saoBlurRadius, - this.params.saoBlurStdDev, - new Vector2(1, 0) - ) - this.prevStdDev = this.params.saoBlurStdDev - this.prevNumSamples = this.params.saoBlurRadius - } - - const restoreVisibility = this.batcher.saveVisiblity() - const opaque = this.batcher.getOpaque() - this.batcher.applyVisibility(opaque) - // Re-render scene if depth texture extension is not supported - if (!this.supportsDepthTextureExtension) { - // Clear rule : far clipping plane in both RGBA and Basic encoding - this.renderOverride( - renderer, - this.depthMaterial, - this.depthRenderTarget, - 0x000000, - 1.0 - ) - } - - if (this.normalsType === NormalsType.DEFAULT) { - if (this.supportsNormalTexture) { - // Clear rule : default normal is facing the camera - this.renderOverride( - renderer, - this.normalMaterial, - this.normalRenderTarget, - 0x7777ff, - 1.0 - ) - } - } - this.batcher.applyVisibility(restoreVisibility) + // const restoreVisibility = this.batcher.saveVisiblity() + // const opaque = this.batcher.getOpaque() + // this.batcher.applyVisibility(opaque) + // this.batcher.applyVisibility(restoreVisibility) // Rendering SAO texture - this.renderPass(renderer, this.saoMaterial, this.saoRenderTarget, 0x000000, 1.0) - - // Blurring SAO texture - if (this.params.saoBlur) { - this.renderPass( - renderer, - this.vBlurMaterial, - this.blurIntermediateRenderTarget, - 0xffffff, - 1.0 - ) - this.renderPass(renderer, this.hBlurMaterial, this.saoRenderTarget, 0xffffff, 1.0) - } - } - - public renderPass( - renderer, - passMaterial, - renderTarget, - clearColor = undefined, - clearAlpha = undefined - ) { - // save original state - renderer.getClearColor(this.originalClearColor) + renderer.getClearColor(this.colorBuffer) const originalClearAlpha = renderer.getClearAlpha() const originalAutoClear = renderer.autoClear - renderer.setRenderTarget(renderTarget) + renderer.setRenderTarget(this.saoRenderTarget) // setup pass state renderer.autoClear = false - if (clearColor !== undefined && clearColor !== null) { - renderer.setClearColor(clearColor) - renderer.setClearAlpha(clearAlpha || 0.0) + renderer.setClearColor(0xffffff) + renderer.setClearAlpha(1.0) + renderer.clear() + this.fsQuad.material = this.saoMaterial + this.fsQuad.render(renderer) + + if (this.params.blurEnabled) { + renderer.setRenderTarget(this.blurIntermediateRenderTarget) + renderer.setClearColor(0xffffff) + renderer.setClearAlpha(1.0) renderer.clear() + this.fsQuad.material = this.vBlurMaterial + this.fsQuad.render(renderer) + + renderer.setRenderTarget(this.saoRenderTarget) + this.fsQuad.material = this.hBlurMaterial + this.fsQuad.render(renderer) } - ;(this.fsQuad as FullScreenQuad).material = passMaterial - ;(this.fsQuad as FullScreenQuad).render(renderer) // restore original state renderer.autoClear = originalAutoClear - renderer.setClearColor(this.originalClearColor) + renderer.setClearColor(this.colorBuffer) renderer.setClearAlpha(originalClearAlpha) } - public renderOverride( - renderer, - overrideMaterial, - renderTarget, - clearColor, - clearAlpha - ) { - renderer.getClearColor(this.originalClearColor) - const originalClearAlpha = renderer.getClearAlpha() - const originalAutoClear = renderer.autoClear + public setSize(width: number, height: number) { + this.saoRenderTarget.setSize(width, height) + this.blurIntermediateRenderTarget.setSize(width, height) - renderer.setRenderTarget(renderTarget) - renderer.autoClear = false - - clearColor = overrideMaterial.clearColor || clearColor - clearAlpha = overrideMaterial.clearAlpha || clearAlpha - if (clearColor !== undefined && clearColor !== null) { - renderer.setClearColor(clearColor) - renderer.setClearAlpha(clearAlpha || 0.0) - renderer.clear() - } - - const shadowmapEnabled = renderer.shadowMap.enabled - const shadowmapNeedsUpdate = renderer.shadowMap.needsUpdate - this.scene.overrideMaterial = overrideMaterial - renderer.shadowMap.enabled = false - renderer.shadowMap.needsUpdate = false - renderer.render(this.scene, this.camera) - renderer.shadowMap.enabled = shadowmapEnabled - renderer.shadowMap.needsUpdate = shadowmapNeedsUpdate - this.scene.overrideMaterial = null - - // restore original state - renderer.autoClear = originalAutoClear - renderer.setClearColor(this.originalClearColor) - renderer.setClearAlpha(originalClearAlpha) + this.saoMaterial.uniforms['size'].value.set(width, height) + this.vBlurMaterial.uniforms['size'].value.set(width, height) + this.hBlurMaterial.uniforms['size'].value.set(width, height) + this.saoMaterial.needsUpdate = true } } diff --git a/packages/viewer/src/modules/pipeline/SpeckleStaticAOGeneratePass.ts b/packages/viewer/src/modules/pipeline/SpeckleStaticAOGeneratePass.ts index 3d98cacd3..315ed41b0 100644 --- a/packages/viewer/src/modules/pipeline/SpeckleStaticAOGeneratePass.ts +++ b/packages/viewer/src/modules/pipeline/SpeckleStaticAOGeneratePass.ts @@ -174,7 +174,6 @@ export class SpeckleStaticAOGeneratePass extends Pass { public render(renderer, writeBuffer, readBuffer) { writeBuffer readBuffer - // save original state const originalClearColor = new Color() renderer.getClearColor(originalClearColor) From b5cfe00c04edafb2ca5ff91d009293e255661ea3 Mon Sep 17 00:00:00 2001 From: AlexandruPopovici Date: Fri, 14 Oct 2022 00:36:33 +0300 Subject: [PATCH 07/54] A lot of changes. Pipeline rework is more or less complete. Added individual output for some pipeline passes that we might want to inspect inside the sandbox. --- packages/viewer-sandbox/src/Sandbox.ts | 261 ++++++++++-------- .../viewer/src/modules/SpeckleRenderer.ts | 55 ++-- packages/viewer/src/modules/Viewer.ts | 4 +- .../shaders/speckle-copy-output-frag.ts | 30 ++ .../shaders/speckle-copy-output-vert.ts | 6 + .../materials/shaders/speckle-sao-frag.ts | 15 + .../src/modules/pipeline/ApplySAOPass.ts | 29 +- .../src/modules/pipeline/CopyOutputPass.ts | 56 ++++ .../viewer/src/modules/pipeline/DepthPass.ts | 14 +- ...kleDynamicSAOPass.ts => DynamicSAOPass.ts} | 58 ++-- .../src/modules/pipeline/NormalsPass.ts | 95 +++++++ .../viewer/src/modules/pipeline/Pipeline.ts | 200 +++++++++----- .../src/modules/pipeline/SpecklePass.ts | 17 ++ 13 files changed, 583 insertions(+), 257 deletions(-) create mode 100644 packages/viewer/src/modules/materials/shaders/speckle-copy-output-frag.ts create mode 100644 packages/viewer/src/modules/materials/shaders/speckle-copy-output-vert.ts create mode 100644 packages/viewer/src/modules/pipeline/CopyOutputPass.ts rename packages/viewer/src/modules/pipeline/{SpeckleDynamicSAOPass.ts => DynamicSAOPass.ts} (90%) create mode 100644 packages/viewer/src/modules/pipeline/NormalsPass.ts create mode 100644 packages/viewer/src/modules/pipeline/SpecklePass.ts diff --git a/packages/viewer-sandbox/src/Sandbox.ts b/packages/viewer-sandbox/src/Sandbox.ts index 3a8eeb973..1dadd4c7e 100644 --- a/packages/viewer-sandbox/src/Sandbox.ts +++ b/packages/viewer-sandbox/src/Sandbox.ts @@ -30,26 +30,20 @@ export default class Sandbox { tonemapping: 4 //'ACESFilmicToneMapping' } - public static postParams = { - saoEnabled: true, - saoParams: { - saoBias: 0.15, - saoIntensity: 1.25, - saoScale: 434, - saoKernelRadius: 15, - saoMinResolution: 0, - saoBlur: true, - saoBlurRadius: 4, - saoBlurStdDev: 4, - saoBlurDepthCutoff: 0.0007 - }, - saoScaleOffset: 0, - saoNormalsRendering: 2, - minDistance: 0.0, - maxDistance: 0.008, - ssaoKernelRadius: 0.5, - progressiveAO: 0, - progressive: true + public static pipelineParams = { + pipelineOutput: 8, + dynamicAoEnabled: true, + dynamicAoParams: { + intensity: 1.25, + scale: 0, + kernelRadius: 10, + bias: 0.15, + normalsType: 2, + blurEnabled: true, + blurRadius: 4, + blurStdDev: 4, + blurDepthCutoff: 0.0007 + } } public static lightParams: SunLightConfiguration = { @@ -343,33 +337,88 @@ export default class Sandbox { this.viewer.getRenderer().renderer.toneMapping = Sandbox.sceneParams.tonemapping this.viewer.requestRender() }) - postFolder - .addInput(Sandbox.postParams, 'saoEnabled', { label: 'SAO-ENABLED' }) - .on('change', () => { - this.viewer.getRenderer().pipelineOptions = Sandbox.postParams - this.viewer.requestRender() - }) - postFolder - .addInput(Sandbox.postParams.saoParams, 'saoBias', { - min: -1, - max: 1 + + const pipelineFolder = this.tabs.pages[1].addFolder({ + title: 'Pipeline', + expanded: true + }) + pipelineFolder + .addInput(Sandbox.pipelineParams, 'pipelineOutput', { + options: { + DEPTH_RGBA: 0, + DEPTH: 1, + COLOR: 2, + GEOMETRY_NORMALS: 3, + RECONSTRUCTED_NORMALS: 4, + DYNAMIC_AO: 5, + DYNAMIC_AO_BLURED: 6, + PROGRESSIVE_AO: 7, + FINAL: 8 + } }) .on('change', () => { - this.viewer.getRenderer().pipelineOptions = Sandbox.postParams - this.viewer.requestRender() - }) - postFolder - .addInput(Sandbox.postParams.saoParams, 'saoIntensity', { - min: 0, - max: 5 - }) - .on('change', () => { - this.viewer.getRenderer().pipelineOptions = Sandbox.postParams + this.viewer.getRenderer().pipelineOptions = Sandbox.pipelineParams this.viewer.requestRender() }) + // postFolder + // .addInput(Sandbox.postParams, 'saoEnabled', { label: 'SAO-ENABLED' }) + // .on('change', () => { + // this.viewer.getRenderer().pipelineOptions = Sandbox.postParams + // this.viewer.requestRender() + // }) + // postFolder + // .addInput(Sandbox.postParams.saoParams, 'saoBias', { + // min: -1, + // max: 1 + // }) + // .on('change', () => { + // this.viewer.getRenderer().pipelineOptions = Sandbox.postParams + // this.viewer.requestRender() + // }) + // postFolder + // .addInput(Sandbox.postParams.saoParams, 'saoIntensity', { + // min: 0, + // max: 5 + // }) + // .on('change', () => { + // this.viewer.getRenderer().pipelineOptions = Sandbox.postParams + // this.viewer.requestRender() + // }) + + // // postFolder + // // .addInput(Sandbox.postParams.saoParams, 'saoScale', { + // // min: 0, + // // max: 100 + // // }) + // // .on('change', () => { + // // this.viewer.getRenderer().pipelineOptions = Sandbox.postParams + // // this.viewer.requestRender() + // // }) + // postFolder + // .addInput(Sandbox.postParams, 'saoScaleOffset', { + // min: -100, + // max: 100 + // }) + // .on('change', () => { + // this.viewer.getRenderer().pipelineOptions = Sandbox.postParams + // this.viewer.requestRender() + // }) // postFolder - // .addInput(Sandbox.postParams.saoParams, 'saoScale', { + // .addInput(Sandbox.postParams, 'saoNormalsRendering', { + // options: { + // DEFAULT: 0, + // ADVANCED: 1, + // ACCURATE: 2 + // } + // }) + // .on('change', () => { + // this.viewer.getRenderer().pipelineOptions = Sandbox.postParams + // this.viewer.requestRender() + // }) + + // postFolder + // .addInput(Sandbox.postParams.saoParams, 'saoKernelRadius', { // min: 0, // max: 100 // }) @@ -377,97 +426,65 @@ export default class Sandbox { // this.viewer.getRenderer().pipelineOptions = Sandbox.postParams // this.viewer.requestRender() // }) - postFolder - .addInput(Sandbox.postParams, 'saoScaleOffset', { - min: -100, - max: 100 - }) - .on('change', () => { - this.viewer.getRenderer().pipelineOptions = Sandbox.postParams - this.viewer.requestRender() - }) - postFolder - .addInput(Sandbox.postParams, 'saoNormalsRendering', { - options: { - DEFAULT: 0, - ADVANCED: 1, - ACCURATE: 2 - } - }) - .on('change', () => { - this.viewer.getRenderer().pipelineOptions = Sandbox.postParams - this.viewer.requestRender() - }) - - postFolder - .addInput(Sandbox.postParams.saoParams, 'saoKernelRadius', { - min: 0, - max: 100 - }) - .on('change', () => { - this.viewer.getRenderer().pipelineOptions = Sandbox.postParams - this.viewer.requestRender() - }) + // // postFolder + // // .addInput(Sandbox.postParams.saoParams, 'saoMinResolution', { + // // min: 0, + // // max: 1 + // // }) + // // .on('change', () => { + // // this.viewer.getRenderer().pipelineOptions = Sandbox.postParams + // // this.viewer.requestRender() + // // }) // postFolder - // .addInput(Sandbox.postParams.saoParams, 'saoMinResolution', { - // min: 0, - // max: 1 - // }) + // .addInput(Sandbox.postParams.saoParams, 'saoBlur', {}) // .on('change', () => { // this.viewer.getRenderer().pipelineOptions = Sandbox.postParams // this.viewer.requestRender() // }) - postFolder - .addInput(Sandbox.postParams.saoParams, 'saoBlur', {}) - .on('change', () => { - this.viewer.getRenderer().pipelineOptions = Sandbox.postParams - this.viewer.requestRender() - }) + // postFolder + // .addInput(Sandbox.postParams.saoParams, 'saoBlurRadius', { min: 0, max: 10 }) + // .on('change', () => { + // this.viewer.getRenderer().pipelineOptions = Sandbox.postParams + // this.viewer.requestRender() + // }) - postFolder - .addInput(Sandbox.postParams.saoParams, 'saoBlurRadius', { min: 0, max: 10 }) - .on('change', () => { - this.viewer.getRenderer().pipelineOptions = Sandbox.postParams - this.viewer.requestRender() - }) + // postFolder + // .addInput(Sandbox.postParams, 'minDistance', { min: 0, max: 100, step: 0.000001 }) + // .on('change', () => { + // this.viewer.getRenderer().pipelineOptions = Sandbox.postParams + // this.viewer.requestRender() + // }) - postFolder - .addInput(Sandbox.postParams, 'minDistance', { min: 0, max: 100, step: 0.000001 }) - .on('change', () => { - this.viewer.getRenderer().pipelineOptions = Sandbox.postParams - this.viewer.requestRender() - }) - - postFolder - .addInput(Sandbox.postParams, 'maxDistance', { min: 0, max: 100, step: 0.000001 }) - .on('change', () => { - this.viewer.getRenderer().pipelineOptions = Sandbox.postParams - this.viewer.requestRender() - }) - postFolder - .addInput(Sandbox.postParams, 'ssaoKernelRadius', { min: 0, max: 100 }) - .on('change', () => { - this.viewer.getRenderer().pipelineOptions = Sandbox.postParams - this.viewer.requestRender() - }) - postFolder.addInput(Sandbox.postParams, 'progressive', {}).on('change', () => { - this.viewer.getRenderer().pipelineOptions = Sandbox.postParams - this.viewer.requestRender() - }) - postFolder - .addInput(Sandbox.postParams, 'progressiveAO', { - options: { - SAO: 0, - SSAO: 1 - } - }) - .on('change', () => { - this.viewer.getRenderer().pipelineOptions = Sandbox.postParams - this.viewer.requestRender() - }) + // postFolder + // .addInput(Sandbox.postParams, 'maxDistance', { min: 0, max: 100, step: 0.000001 }) + // .on('change', () => { + // this.viewer.getRenderer().pipelineOptions = Sandbox.postParams + // this.viewer.requestRender() + // }) + // postFolder + // .addInput(Sandbox.postParams, 'ssaoKernelRadius', { min: 0, max: 100 }) + // .on('change', () => { + // this.viewer.getRenderer().pipelineOptions = Sandbox.postParams + // this.viewer.requestRender() + // }) + // postFolder.addInput(Sandbox.postParams, 'progressive', {}).on('change', () => { + // this.viewer.getRenderer().pipelineOptions = Sandbox.postParams + // this.viewer.requestRender() + // }) + // postFolder + // .addInput(Sandbox.postParams, 'progressiveAO', { + // options: { + // SAO: 0, + // SSAO: 1 + // } + // }) + // .on('change', () => { + // this.viewer.getRenderer().pipelineOptions = Sandbox.postParams + // this.viewer.requestRender() + // }) // postFolder // .addInput(Sandbox.postParams.saoParams, 'saoBlurStdDev', { diff --git a/packages/viewer/src/modules/SpeckleRenderer.ts b/packages/viewer/src/modules/SpeckleRenderer.ts index 4c18d0ff8..0f01cd9e8 100644 --- a/packages/viewer/src/modules/SpeckleRenderer.ts +++ b/packages/viewer/src/modules/SpeckleRenderer.ts @@ -50,7 +50,7 @@ import { DefaultPipelineOptions, Pipeline, PipelineOptions } from './pipeline/Pi export default class SpeckleRenderer { private readonly SHOW_HELPERS = false private _renderer: WebGLRenderer - public scene: Scene + public _scene: Scene private rootGroup: Group private batcher: Batcher private intersections: Intersections @@ -62,19 +62,12 @@ export default class SpeckleRenderer { private filterBatchRecording: string[] private pipeline: Pipeline - private lastAzimuth: number - private lastPolar: number - private lastDistance: number - private lastTarget: Vector3 = new Vector3() - private lasMaxCameraMotion: number - private readonly CAMERA_MOTION_EPSILON: number = 0.0001 - public get renderer(): WebGLRenderer { return this._renderer } public set indirectIBL(texture: Texture) { - this.scene.environment = texture + this._scene.environment = texture } public set indirectIBLIntensity(value: number) { @@ -92,11 +85,11 @@ export default class SpeckleRenderer { /** TEMPORARY for backwards compatibility */ public get allObjects() { - return this.scene.getObjectByName('ContentGroup') + return this._scene.getObjectByName('ContentGroup') } public subtree(subtreeId: string) { - return this.scene.getObjectByName(subtreeId) + return this._scene.getObjectByName(subtreeId) } public get sceneBox() { @@ -115,15 +108,23 @@ export default class SpeckleRenderer { return this.sun } + public get camera() { + return this.viewer.cameraHandler.activeCam.camera + } + + public get scene() { + return this._scene + } + public set pipelineOptions(value: PipelineOptions) { this.pipeline.pipelineOptions = value } public constructor(viewer: Viewer /** TEMPORARY */) { - this.scene = new Scene() + this._scene = new Scene() this.rootGroup = new Group() this.rootGroup.name = 'ContentGroup' - this.scene.add(this.rootGroup) + this._scene.add(this.rootGroup) this.batcher = new Batcher() this.intersections = new Intersections() @@ -151,7 +152,7 @@ export default class SpeckleRenderer { container.appendChild(this._renderer.domElement) this.pipeline = new Pipeline(this._renderer, this.batcher) - this.pipeline.configure(this.scene, this.viewer.cameraHandler.activeCam.camera) + this.pipeline.configure(this._scene, this.viewer.cameraHandler.activeCam.camera) this.pipeline.pipelineOptions = DefaultPipelineOptions this.input = new Input(this._renderer.domElement, InputOptionsDefault) @@ -163,7 +164,7 @@ export default class SpeckleRenderer { if (this.SHOW_HELPERS) { const helpers = new Group() helpers.name = 'Helpers' - this.scene.add(helpers) + this._scene.add(helpers) const sceneBoxHelper = new Box3Helper(this.sceneBox, new Color(0x0000ff)) sceneBoxHelper.name = 'SceneBoxHelper' @@ -287,7 +288,8 @@ export default class SpeckleRenderer { this.viewer.cameraHandler.activeCam.camera.far = d this.viewer.cameraHandler.activeCam.camera.updateProjectionMatrix() this.viewer.cameraHandler.camera.updateProjectionMatrix() - this.pipeline.pipelineOptions = { saoParams: { saoScale: d } } + + this.pipeline.update(this) // const currentAzimuth = this.viewer.cameraHandler.controls.azimuthAngle // const currentPolar = this.viewer.cameraHandler.controls.polarAngle @@ -321,8 +323,9 @@ export default class SpeckleRenderer { } public render(camera: Camera): boolean { + camera this.batcher.render(this.renderer) - const needsRender = this.pipeline.render(this.scene, camera) + const needsRender = this.pipeline.render() return needsRender // this.renderer.render(this.scene, camera) } @@ -431,7 +434,7 @@ export default class SpeckleRenderer { private addDirectLights() { this.sun = new DirectionalLight(0xffffff, 5) this.sun.name = 'sun' - this.scene.add(this.sun) + this._scene.add(this.sun) this.sun.castShadow = true @@ -450,7 +453,7 @@ export default class SpeckleRenderer { this.sun.shadow.radius = 2 this.sunTarget = new Object3D() - this.scene.add(this.sunTarget) + this._scene.add(this.sunTarget) this.sunTarget.position.copy(this.sceneCenter) this.sun.target = this.sunTarget } @@ -529,12 +532,14 @@ export default class SpeckleRenderer { public updateHelpers() { if (this.SHOW_HELPERS) { - ;(this.scene.getObjectByName('CamHelper') as CameraHelper).update() + ;(this._scene.getObjectByName('CamHelper') as CameraHelper).update() // Thank you prettier, this looks so much better - ;(this.scene.getObjectByName('SceneBoxHelper') as Box3Helper).box.copy( + ;(this._scene.getObjectByName('SceneBoxHelper') as Box3Helper).box.copy( this.sceneBox ) - ;(this.scene.getObjectByName('DirLightHelper') as DirectionalLightHelper).update() + ;( + this._scene.getObjectByName('DirLightHelper') as DirectionalLightHelper + ).update() } } @@ -575,7 +580,7 @@ export default class SpeckleRenderer { private onObjectClick(e) { const results: Array = this.intersections.intersect( - this.scene, + this._scene, this.viewer.cameraHandler.activeCam.camera, e, true, @@ -614,7 +619,7 @@ export default class SpeckleRenderer { private onObjectDoubleClick(e) { const results: Array = this.intersections.intersect( - this.scene, + this._scene, this.viewer.cameraHandler.activeCam.camera, e, true, @@ -899,7 +904,7 @@ export default class SpeckleRenderer { /** DEBUG */ public onObjectClickDebug(e) { const results: Array = this.intersections.intersect( - this.scene, + this._scene, this.viewer.cameraHandler.activeCam.camera, e, true, diff --git a/packages/viewer/src/modules/Viewer.ts b/packages/viewer/src/modules/Viewer.ts index 8e816a122..d474201cd 100644 --- a/packages/viewer/src/modules/Viewer.ts +++ b/packages/viewer/src/modules/Viewer.ts @@ -135,7 +135,9 @@ export class Viewer extends EventEmitter implements IViewer { } public resize() { - this.speckleRenderer.resize(this.container.offsetWidth, this.container.offsetHeight) + const width = this.container.offsetWidth + const height = this.container.offsetHeight + this.speckleRenderer.resize(width, height) this.needsRender = true } diff --git a/packages/viewer/src/modules/materials/shaders/speckle-copy-output-frag.ts b/packages/viewer/src/modules/materials/shaders/speckle-copy-output-frag.ts new file mode 100644 index 000000000..f696ba58d --- /dev/null +++ b/packages/viewer/src/modules/materials/shaders/speckle-copy-output-frag.ts @@ -0,0 +1,30 @@ +export const speckleCopyOutputFrag = ` + uniform float opacity; + uniform sampler2D tDiffuse; + varying vec2 vUv; + + const float UnpackDownscale = 255. / 256.; + const vec3 PackFactors = vec3( 256. * 256. * 256., 256. * 256., 256. ); + const vec4 UnpackFactors = UnpackDownscale / vec4( PackFactors, 1. ); + + float unpackRGBAToDepth( const in vec4 v ) { + return dot( v, UnpackFactors ); + } + + vec3 unpackRGBToNormal( const in vec3 rgb ) { + return 2.0 * rgb.xyz - 1.0; + } + + void main() { + vec4 inSample = texture2D( tDiffuse, vUv ); + vec3 outSample = inSample.rgb; + #if OUTPUT_TYPE == 1 + outSample.rgb = vec3(unpackRGBAToDepth(inSample)); + #endif + #if OUTPUT_TYPE == 3 + outSample.rgb = unpackRGBToNormal(inSample.rgb); + #endif + + gl_FragColor.rgb = outSample; + gl_FragColor.a = 1.; + }` diff --git a/packages/viewer/src/modules/materials/shaders/speckle-copy-output-vert.ts b/packages/viewer/src/modules/materials/shaders/speckle-copy-output-vert.ts new file mode 100644 index 000000000..7a083de37 --- /dev/null +++ b/packages/viewer/src/modules/materials/shaders/speckle-copy-output-vert.ts @@ -0,0 +1,6 @@ +export const speckleCopyOutputVert = ` + varying vec2 vUv; + void main() { + vUv = uv; + gl_Position = projectionMatrix * modelViewMatrix * vec4( position, 1.0 ); + }` diff --git a/packages/viewer/src/modules/materials/shaders/speckle-sao-frag.ts b/packages/viewer/src/modules/materials/shaders/speckle-sao-frag.ts index 53dc6b550..66293cd88 100644 --- a/packages/viewer/src/modules/materials/shaders/speckle-sao-frag.ts +++ b/packages/viewer/src/modules/materials/shaders/speckle-sao-frag.ts @@ -209,6 +209,21 @@ export const speckleSaoFrag = /* glsl */ ` } float centerViewZ = getViewZ( centerDepth ); vec3 viewPosition = getViewPosition( vUv, centerDepth, centerViewZ ); + + #ifdef OUTPUT_RECONSTRUCTED_NORMALS + vec3 normal; + #if IMPROVED_NORMAL_RECONSTRUCTION == 1 + normal = viewNormalImproved(vUv, viewPosition); + #elif ACCURATE_NORMAL_RECONSTRUCTION == 1 + normal = viewNormalAccurate(vUv, viewPosition, centerDepth); + #else + normal = normalize( cross( dFdx( viewPosition ), dFdy( viewPosition ) ) ); + #endif + gl_FragColor.rgb = packNormalToRGB(viewNormalImproved(vUv, viewPosition)); + gl_FragColor.a = 1.; + return; + #endif + float ambientOcclusion = getAmbientOcclusion( viewPosition, centerDepth ); gl_FragColor = getDefaultColor( vUv ); gl_FragColor.xyz *= 1. - ambientOcclusion; diff --git a/packages/viewer/src/modules/pipeline/ApplySAOPass.ts b/packages/viewer/src/modules/pipeline/ApplySAOPass.ts index 65b2cf55e..a378f3c45 100644 --- a/packages/viewer/src/modules/pipeline/ApplySAOPass.ts +++ b/packages/viewer/src/modules/pipeline/ApplySAOPass.ts @@ -11,8 +11,9 @@ import { } from 'three' import { FullScreenQuad, Pass } from 'three/examples/jsm/postprocessing/Pass' import { CopyShader } from 'three/examples/jsm/shaders/CopyShader.js' +import { InputColorTextureUniform, SpecklePass } from './SpecklePass' -export class ApplySAOPass extends Pass { +export class ApplySAOPass extends Pass implements SpecklePass { private fsQuad: FullScreenQuad public materialCopy: ShaderMaterial @@ -35,26 +36,34 @@ export class ApplySAOPass extends Pass { this.materialCopy.blendDstAlpha = ZeroFactor this.materialCopy.blendEquationAlpha = AddEquation - // this.materialCopy.blending = CustomBlending - // this.materialCopy.blendSrc = OneFactor - // this.materialCopy.blendDst = OneFactor - // this.materialCopy.blendEquation = ReverseSubtractEquation - // this.materialCopy.blendSrcAlpha = OneFactor - // this.materialCopy.blendDstAlpha = OneFactor - // this.materialCopy.blendEquationAlpha = AddEquation this.materialCopy.needsUpdate = true this.fsQuad = new FullScreenQuad(this.materialCopy) } - public setAoTexture(texture: Texture) { - this.materialCopy.uniforms['tDiffuse'].value = texture + public setTexture(uName: InputColorTextureUniform, texture: Texture) { + this.materialCopy.uniforms[uName].value = texture this.materialCopy.needsUpdate = true } + get displayName(): string { + return 'APPLYSAO' + } + + get outputTexture(): Texture { + return null + } + + setParams(params: unknown) { + params + } + render(renderer, writeBuffer, readBuffer /*, deltaTime, maskActive*/) { writeBuffer readBuffer renderer.setRenderTarget(null) + const rendereAutoClear = renderer.autoClear + renderer.autoClear = false this.fsQuad.render(renderer) + renderer.autoClear = rendereAutoClear } } diff --git a/packages/viewer/src/modules/pipeline/CopyOutputPass.ts b/packages/viewer/src/modules/pipeline/CopyOutputPass.ts new file mode 100644 index 000000000..076b213e0 --- /dev/null +++ b/packages/viewer/src/modules/pipeline/CopyOutputPass.ts @@ -0,0 +1,56 @@ +import { NoBlending, ShaderMaterial, Texture, UniformsUtils } from 'three' +import { FullScreenQuad, Pass } from 'three/examples/jsm/postprocessing/Pass' +import { CopyShader } from 'three/examples/jsm/shaders/CopyShader.js' +import { speckleCopyOutputFrag } from '../materials/shaders/speckle-copy-output-frag' +import { speckleCopyOutputVert } from '../materials/shaders/speckle-copy-output-vert' +import { PipelineOutputType } from './Pipeline' +import { InputColorTextureUniform, SpecklePass } from './SpecklePass' + +export class CopyOutputPass extends Pass implements SpecklePass { + private fsQuad: FullScreenQuad + public materialCopy: ShaderMaterial + + constructor() { + super() + this.materialCopy = new ShaderMaterial({ + defines: { + INPUT_TYPE: 0 + }, + uniforms: UniformsUtils.clone(CopyShader.uniforms), + vertexShader: speckleCopyOutputVert, + fragmentShader: speckleCopyOutputFrag, + blending: NoBlending + }) + + this.materialCopy.needsUpdate = true + this.fsQuad = new FullScreenQuad(this.materialCopy) + } + + public setOutputType(type: PipelineOutputType) { + this.materialCopy.defines['OUTPUT_TYPE'] = type + this.materialCopy.needsUpdate = true + } + + public setTexture(uName: InputColorTextureUniform, texture: Texture) { + this.materialCopy.uniforms[uName].value = texture + this.materialCopy.needsUpdate = true + } + + get displayName(): string { + return 'COPY-OUTPUT' + } + + get outputTexture(): Texture { + return null + } + + render(renderer, writeBuffer, readBuffer /*, deltaTime, maskActive*/) { + writeBuffer + readBuffer + renderer.setRenderTarget(null) + const rendereAutoClear = renderer.autoClear + renderer.autoClear = false + this.fsQuad.render(renderer) + renderer.autoClear = rendereAutoClear + } +} diff --git a/packages/viewer/src/modules/pipeline/DepthPass.ts b/packages/viewer/src/modules/pipeline/DepthPass.ts index 6b1397f81..71db4eb07 100644 --- a/packages/viewer/src/modules/pipeline/DepthPass.ts +++ b/packages/viewer/src/modules/pipeline/DepthPass.ts @@ -3,6 +3,7 @@ import { Color, DoubleSide, NoBlending, + Plane, RGBADepthPacking, Scene, Texture, @@ -10,7 +11,7 @@ import { } from 'three' import { Pass } from 'three/examples/jsm/postprocessing/Pass' import SpeckleDepthMaterial from '../materials/SpeckleDepthMaterial' -import { SpecklePass } from './Pipeline' +import { SpecklePass } from './SpecklePass' export class DepthPass extends Pass implements SpecklePass { private renderTarget: WebGLRenderTarget @@ -20,6 +21,9 @@ export class DepthPass extends Pass implements SpecklePass { private colorBuffer: Color = new Color() + public onBeforeRender: () => void = null + public onAfterRender: () => void = null + get displayName(): string { return 'DEPTH' } @@ -49,7 +53,11 @@ export class DepthPass extends Pass implements SpecklePass { this.depthMaterial.side = DoubleSide } - public update(camera: Camera, scene: Scene) { + public setClippingPlanes(planes: Plane[]) { + this.depthMaterial.clippingPlanes = planes + } + + public update(scene: Scene, camera: Camera) { this.camera = camera this.scene = scene } @@ -58,6 +66,7 @@ export class DepthPass extends Pass implements SpecklePass { writeBuffer readBuffer + this.onBeforeRender() renderer.getClearColor(this.colorBuffer) const originalClearAlpha = renderer.getClearAlpha() const originalAutoClear = renderer.autoClear @@ -83,6 +92,7 @@ export class DepthPass extends Pass implements SpecklePass { renderer.autoClear = originalAutoClear renderer.setClearColor(this.colorBuffer) renderer.setClearAlpha(originalClearAlpha) + this.onAfterRender() } public setSize(width: number, height: number) { diff --git a/packages/viewer/src/modules/pipeline/SpeckleDynamicSAOPass.ts b/packages/viewer/src/modules/pipeline/DynamicSAOPass.ts similarity index 90% rename from packages/viewer/src/modules/pipeline/SpeckleDynamicSAOPass.ts rename to packages/viewer/src/modules/pipeline/DynamicSAOPass.ts index c29a0c1e1..d29490af6 100644 --- a/packages/viewer/src/modules/pipeline/SpeckleDynamicSAOPass.ts +++ b/packages/viewer/src/modules/pipeline/DynamicSAOPass.ts @@ -17,7 +17,7 @@ import { speckleSaoVert } from '../materials/shaders/speckle-sao-vert' import { SAOShader } from 'three/examples/jsm/shaders/SAOShader.js' import { DepthLimitedBlurShader } from 'three/examples/jsm/shaders/DepthLimitedBlurShader.js' import { BlurShaderUtils } from 'three/examples/jsm/shaders/DepthLimitedBlurShader.js' -import { SpecklePass } from './Pipeline' +import { InputDepthTextureUniform, SpecklePass } from './SpecklePass' export enum NormalsType { DEFAULT = 0, @@ -49,12 +49,8 @@ export const DefaultSpeckleDynamicSAOPassParams = { blurDepthCutoff: 0.0007 } -/** - * SAO implementation inspired from bhouston previous SAO work - */ - -export class SpeckleDynamicSAOPass extends Pass implements SpecklePass { - private params: SpeckleDynamicSAOPassParams +export class DynamicSAOPass extends Pass implements SpecklePass { + private params: SpeckleDynamicSAOPassParams = DefaultSpeckleDynamicSAOPassParams private colorBuffer: Color = new Color() private saoMaterial: ShaderMaterial = null private vBlurMaterial: ShaderMaterial = null @@ -66,32 +62,22 @@ export class SpeckleDynamicSAOPass extends Pass implements SpecklePass { private prevStdDev: number private prevNumSamples: number - get displayName(): string { + public get displayName(): string { return 'SAO' } - get outputTexture(): Texture { + public get outputTexture(): Texture { return this.saoRenderTarget.texture } - public setDepthTexture(texture: Texture) { - this.saoMaterial.uniforms['tDepth'].value = texture - this.vBlurMaterial.uniforms['tDepth'].value = texture - this.hBlurMaterial.uniforms['tDepth'].value = texture - this.saoMaterial.needsUpdate = true - this.vBlurMaterial.needsUpdate = true - this.hBlurMaterial.needsUpdate = true + public set outputReconstructedNormals(value: boolean) { + if (value) this.saoMaterial.defines['OUTPUT_RECONSTRUCTED_NORMALS'] = '' + else delete this.saoMaterial.defines['OUTPUT_RECONSTRUCTED_NORMALS'] } constructor() { super() - // this.normalRenderTarget.depthBuffer = true - // this.normalRenderTarget.stencilBuffer = true - - // this.normalMaterial = new SpeckleNormalMaterial({}, ['USE_RTE']) - // this.normalMaterial.blending = NoBlending - // this.normalMaterial.side = DoubleSide this.saoRenderTarget = new WebGLRenderTarget(256, 256) this.blurIntermediateRenderTarget = new WebGLRenderTarget(256, 256) this.saoMaterial = new ShaderMaterial({ @@ -145,7 +131,21 @@ export class SpeckleDynamicSAOPass extends Pass implements SpecklePass { this.fsQuad = new FullScreenQuad(this.saoMaterial) } + public setParams(params: unknown) { + Object.assign(this.params, params) + } + + public setTexture(uName: InputDepthTextureUniform, texture: Texture) { + this.saoMaterial.uniforms['tDepth'].value = texture + this.vBlurMaterial.uniforms['tDepth'].value = texture + this.hBlurMaterial.uniforms['tDepth'].value = texture + this.saoMaterial.needsUpdate = true + this.vBlurMaterial.needsUpdate = true + this.hBlurMaterial.needsUpdate = true + } + public update(scene: Scene, camera: Camera) { + this.params.scale = (camera as PerspectiveCamera | OrthographicCamera).far /** SAO DEFINES */ this.saoMaterial.defines['PERSPECTIVE_CAMERA'] = (camera as PerspectiveCamera) .isPerspectiveCamera @@ -235,15 +235,9 @@ export class SpeckleDynamicSAOPass extends Pass implements SpecklePass { this.hBlurMaterial.needsUpdate = true } - public render(renderer, writeBuffer, readBuffer) { - writeBuffer - readBuffer - - // const restoreVisibility = this.batcher.saveVisiblity() - // const opaque = this.batcher.getOpaque() - // this.batcher.applyVisibility(opaque) - // this.batcher.applyVisibility(restoreVisibility) - + public render(renderer) { + const outputNormals = + this.saoMaterial.defines['OUTPUT_RECONSTRUCTED_NORMALS'] !== undefined // Rendering SAO texture renderer.getClearColor(this.colorBuffer) const originalClearAlpha = renderer.getClearAlpha() @@ -259,7 +253,7 @@ export class SpeckleDynamicSAOPass extends Pass implements SpecklePass { this.fsQuad.material = this.saoMaterial this.fsQuad.render(renderer) - if (this.params.blurEnabled) { + if (this.params.blurEnabled && !outputNormals) { renderer.setRenderTarget(this.blurIntermediateRenderTarget) renderer.setClearColor(0xffffff) renderer.setClearAlpha(1.0) diff --git a/packages/viewer/src/modules/pipeline/NormalsPass.ts b/packages/viewer/src/modules/pipeline/NormalsPass.ts new file mode 100644 index 000000000..771d0b165 --- /dev/null +++ b/packages/viewer/src/modules/pipeline/NormalsPass.ts @@ -0,0 +1,95 @@ +import { + Camera, + Color, + DoubleSide, + NoBlending, + Plane, + Scene, + Texture, + WebGLRenderTarget +} from 'three' +import { Pass } from 'three/examples/jsm/postprocessing/Pass' +import SpeckleNormalMaterial from '../materials/SpeckleNormalMaterial' +import { SpecklePass } from './SpecklePass' + +export class NormalsPass extends Pass implements SpecklePass { + private renderTarget: WebGLRenderTarget + private normalsMaterial: SpeckleNormalMaterial = null + private scene: Scene + private camera: Camera + + private colorBuffer: Color = new Color() + + public onBeforeRender: () => void = null + public onAfterRender: () => void = null + + get displayName(): string { + return 'GEOMETRY-NORMALS' + } + + get outputTexture(): Texture { + return this.renderTarget.texture + } + + constructor() { + super() + + this.renderTarget = new WebGLRenderTarget(256, 256) + /** On Chromium, on MacOS the 16 bit depth render buffer appears broken. + * We're not really using a stencil buffer at all, we're just forcing + * three.js to use a 24 bit depth render buffer + */ + this.renderTarget.depthBuffer = true + this.renderTarget.stencilBuffer = true + + this.normalsMaterial = new SpeckleNormalMaterial({}, ['USE_RTE']) + this.normalsMaterial.blending = NoBlending + this.normalsMaterial.side = DoubleSide + } + + public setClippingPlanes(planes: Plane[]) { + this.normalsMaterial.clippingPlanes = planes + } + + public update(scene: Scene, camera: Camera) { + this.camera = camera + this.scene = scene + } + + public render(renderer, writeBuffer, readBuffer) { + writeBuffer + readBuffer + + this.onBeforeRender() + renderer.getClearColor(this.colorBuffer) + const originalClearAlpha = renderer.getClearAlpha() + const originalAutoClear = renderer.autoClear + + renderer.setRenderTarget(this.renderTarget) + renderer.autoClear = false + + renderer.setClearColor(0x000000) + renderer.setClearAlpha(1.0) + renderer.clear() + + const shadowmapEnabled = renderer.shadowMap.enabled + const shadowmapNeedsUpdate = renderer.shadowMap.needsUpdate + this.scene.overrideMaterial = this.normalsMaterial + renderer.shadowMap.enabled = false + renderer.shadowMap.needsUpdate = false + renderer.render(this.scene, this.camera) + renderer.shadowMap.enabled = shadowmapEnabled + renderer.shadowMap.needsUpdate = shadowmapNeedsUpdate + this.scene.overrideMaterial = null + + // restore original state + renderer.autoClear = originalAutoClear + renderer.setClearColor(this.colorBuffer) + renderer.setClearAlpha(originalClearAlpha) + this.onAfterRender() + } + + public setSize(width: number, height: number) { + this.renderTarget.setSize(width, height) + } +} diff --git a/packages/viewer/src/modules/pipeline/Pipeline.ts b/packages/viewer/src/modules/pipeline/Pipeline.ts index 86d5ff19c..db2dc1a0e 100644 --- a/packages/viewer/src/modules/pipeline/Pipeline.ts +++ b/packages/viewer/src/modules/pipeline/Pipeline.ts @@ -1,31 +1,40 @@ -import { Camera, Plane, Scene, Texture, Vector2, WebGLRenderer } from 'three' +import { Camera, Plane, Scene, Vector2, WebGLRenderer } from 'three' import { EffectComposer } from 'three/examples/jsm/postprocessing/EffectComposer.js' import { RenderPass } from 'three/examples/jsm/postprocessing/RenderPass.js' import Batcher from '../batching/Batcher' +import SpeckleRenderer from '../SpeckleRenderer' import { ApplySAOPass } from './ApplySAOPass' +import { CopyOutputPass } from './CopyOutputPass' +import { DepthPass } from './DepthPass' +import { NormalsPass } from './NormalsPass' import { DefaultSpeckleDynamicSAOPassParams, - NormalsType, - SpeckleDynamicSAOPass, + DynamicSAOPass, SpeckleDynamicSAOPassParams -} from './SpeckleDynamicSAOPass' -import { SpeckleStaticAOGeneratePass } from './SpeckleStaticAOGeneratePass' +} from './DynamicSAOPass' +// import { SpecklePass } from './SpecklePass' +// import { SpeckleStaticAOGeneratePass } from './SpeckleStaticAOGeneratePass' -enum RenderType { - NORMAL, - ACCUMULATION -} -export interface SpecklePass { - get displayName(): string - get outputTexture(): Texture +export enum PipelineOutputType { + DEPTH_RGBA = 0, + DEPTH = 1, + COLOR = 2, + GEOMETRY_NORMALS = 3, + RECONSTRUCTED_NORMALS = 4, + DYNAMIC_AO = 5, + DYNAMIC_AO_BLURED = 6, + PROGRESSIVE_AO = 7, + FINAL = 8 } export interface PipelineOptions { + pipelineOutput: PipelineOutputType dynamicAoEnabled: boolean dynamicAoParams: SpeckleDynamicSAOPassParams } export const DefaultPipelineOptions: PipelineOptions = { + pipelineOutput: PipelineOutputType.FINAL, dynamicAoEnabled: true, dynamicAoParams: DefaultSpeckleDynamicSAOPassParams // saoScaleOffset: 0, @@ -43,23 +52,79 @@ export class Pipeline { private _pipelineOptions: PipelineOptions = Object.assign({}, DefaultPipelineOptions) private composer: EffectComposer = null + private depthPass: DepthPass = null + private normalsPass: NormalsPass = null private renderPass: RenderPass = null - private dynamicAoPass: SpeckleDynamicSAOPass = null + private dynamicAoPass: DynamicSAOPass = null private applySaoPass: ApplySAOPass = null - private staticAOGenerationPass: SpeckleStaticAOGeneratePass = null + private copyOutputPass: CopyOutputPass = null private drawingSize: Vector2 = new Vector2() - private _renderType: RenderType = RenderType.NORMAL - private accumulationFrame = 0 - private readonly NUM_ACCUMULATION_FRAMES = 16 - private enableProgressive = true public set pipelineOptions(options: Partial) { Object.assign(this._pipelineOptions, options) + this.pipelineOutput = options.pipelineOutput + this.dynamicAoPass.setParams(options.dynamicAoParams) } - private set renderType(value: RenderType) { - this._renderType = value + public set pipelineOutput(outputType: PipelineOutputType) { + switch (outputType) { + case PipelineOutputType.DEPTH_RGBA: + this.dynamicAoPass.enabled = false + this.renderPass.enabled = false + this.applySaoPass.enabled = false + this.normalsPass.enabled = false + this.depthPass.enabled = true + this.copyOutputPass.enabled = true + this.copyOutputPass.setTexture('tDiffuse', this.depthPass.outputTexture) + this.copyOutputPass.setOutputType(PipelineOutputType.DEPTH_RGBA) + break + + case PipelineOutputType.DEPTH: + this.dynamicAoPass.enabled = false + this.renderPass.enabled = false + this.applySaoPass.enabled = false + this.depthPass.enabled = true + this.normalsPass.enabled = false + this.copyOutputPass.enabled = true + this.copyOutputPass.setTexture('tDiffuse', this.depthPass.outputTexture) + this.copyOutputPass.setOutputType(PipelineOutputType.DEPTH) + break + + case PipelineOutputType.COLOR: + this.depthPass.enabled = false + this.dynamicAoPass.enabled = false + this.applySaoPass.enabled = false + this.copyOutputPass.enabled = false + this.normalsPass.enabled = false + this.renderPass.enabled = true + break + + case PipelineOutputType.GEOMETRY_NORMALS: + this.depthPass.enabled = false + this.dynamicAoPass.enabled = false + this.applySaoPass.enabled = false + this.renderPass.enabled = false + this.normalsPass.enabled = true + this.copyOutputPass.enabled = true + this.copyOutputPass.setTexture('tDiffuse', this.normalsPass.outputTexture) + this.copyOutputPass.setOutputType(PipelineOutputType.GEOMETRY_NORMALS) + break + + case PipelineOutputType.RECONSTRUCTED_NORMALS: + this.depthPass.enabled = true + this.dynamicAoPass.enabled = true + this.applySaoPass.enabled = false + this.renderPass.enabled = false + this.normalsPass.enabled = false + this.copyOutputPass.enabled = true + this.copyOutputPass.setTexture('tDiffuse', this.dynamicAoPass.outputTexture) + this.copyOutputPass.setOutputType(PipelineOutputType.GEOMETRY_NORMALS) + this.dynamicAoPass.outputReconstructedNormals = true + break + default: + break + } } public constructor(renderer: WebGLRenderer, batcher: Batcher) { @@ -71,59 +136,64 @@ export class Pipeline { } public configure(scene: Scene, camera: Camera) { - this.dynamicAoPass = new SpeckleDynamicSAOPass( - scene, - camera, - this._batcher, - false, - NormalsType.IMPROVED - ) - this.staticAOGenerationPass = new SpeckleStaticAOGeneratePass(this._batcher) - this.staticAOGenerationPass.depthTexture = - this.dynamicAoPass.depthRenderTarget.texture - this.composer.addPass(this.dynamicAoPass) + this.depthPass = new DepthPass() + this.normalsPass = new NormalsPass() + this.normalsPass.enabled = false + this.dynamicAoPass = new DynamicSAOPass() this.renderPass = new RenderPass(scene, camera) this.renderPass.renderToScreen = true - // this.renderPass.enabled = false - this.composer.addPass(this.renderPass) - this.composer.addPass(this.staticAOGenerationPass) this.applySaoPass = new ApplySAOPass() - this.applySaoPass.setAoTexture(this.dynamicAoPass.saoRenderTarget.texture) this.applySaoPass.renderToScreen = true + this.copyOutputPass = new CopyOutputPass() + this.copyOutputPass.renderToScreen = true + this.copyOutputPass.enabled = false + this.composer.addPass(this.depthPass) + this.composer.addPass(this.normalsPass) + this.composer.addPass(this.dynamicAoPass) + this.composer.addPass(this.renderPass) this.composer.addPass(this.applySaoPass) + this.composer.addPass(this.copyOutputPass) + + this.dynamicAoPass.setTexture('tDepth', this.depthPass.outputTexture) + this.applySaoPass.setTexture('tDiffuse', this.dynamicAoPass.outputTexture) + + let restoreVisibility + this.depthPass.onBeforeRender = () => { + restoreVisibility = this._batcher.saveVisiblity() + const opaque = this._batcher.getOpaque() + this._batcher.applyVisibility(opaque) + } + this.depthPass.onAfterRender = () => { + this._batcher.applyVisibility(restoreVisibility) + } + + this.normalsPass.onBeforeRender = () => { + restoreVisibility = this._batcher.saveVisiblity() + const opaque = this._batcher.getOpaque() + this._batcher.applyVisibility(opaque) + } + this.normalsPass.onAfterRender = () => { + this._batcher.applyVisibility(restoreVisibility) + } } public updateClippingPlanes(planes: Plane[]) { - this.dynamicAoPass.depthMaterial.clippingPlanes = planes - this.dynamicAoPass.normalMaterial.clippingPlanes = planes + this.depthPass.setClippingPlanes(planes) } - public render(scene: Scene, camera: Camera): boolean { + public update(renderer: SpeckleRenderer) { + this.depthPass.update(renderer.scene, renderer.camera) + this.dynamicAoPass.update(renderer.scene, renderer.camera) + this.normalsPass.update(renderer.scene, renderer.camera) + } + + public render(): boolean { this._renderer.getDrawingBufferSize(this.drawingSize) if (this.drawingSize.length() === 0) return - if (this._renderType === RenderType.NORMAL) { - this._renderer.clear(true) - this.applySaoPass.setAoTexture(this.dynamicAoPass.saoRenderTarget.texture) - this.renderPass.scene = scene - this.renderPass.camera = camera - this.dynamicAoPass.scene = scene - this.dynamicAoPass.camera = camera - this.composer.render() - return true - } else if (this.enableProgressive) { - this._renderer.clear(true) - this.applySaoPass.setAoTexture(this.staticAOGenerationPass.outputTexture.texture) - this.renderPass.scene = scene - this.renderPass.camera = camera - this.dynamicAoPass.scene = scene - this.dynamicAoPass.camera = camera - this.staticAOGenerationPass.update(camera, this.accumulationFrame) - this.composer.render() - this.accumulationFrame++ - console.warn('rendering stationary frame => ', this.accumulationFrame) - return this.accumulationFrame < this.NUM_ACCUMULATION_FRAMES ? true : false - } + this._renderer.clear(true) + this.composer.render() + return true } public resize(width: number, height: number) { @@ -131,15 +201,15 @@ export class Pipeline { } public onStationaryBegin() { - this.renderType = RenderType.ACCUMULATION - this.staticAOGenerationPass.enabled = true - this.accumulationFrame = 0 + // this.renderType = RenderType.ACCUMULATION + // this.staticAOGenerationPass.enabled = true + // this.accumulationFrame = 0 console.warn('Starting stationary') } public onStationaryEnd() { - this.renderType = RenderType.NORMAL - this.staticAOGenerationPass.enabled = false + // this.renderType = RenderType.NORMAL + // this.staticAOGenerationPass.enabled = false console.warn('Ending stationary') } } diff --git a/packages/viewer/src/modules/pipeline/SpecklePass.ts b/packages/viewer/src/modules/pipeline/SpecklePass.ts new file mode 100644 index 000000000..0451fcb50 --- /dev/null +++ b/packages/viewer/src/modules/pipeline/SpecklePass.ts @@ -0,0 +1,17 @@ +import { Camera, Plane, Scene, Texture } from 'three' + +export type InputColorTextureUniform = 'tDiffuse' +export type InputDepthTextureUniform = 'tDepth' + +export interface SpecklePass { + onBeforeRender?: () => void + onAferRender?: () => void + + get displayName(): string + get outputTexture(): Texture + + update?(scene: Scene, camera: Camera) + setTexture?(uName: string, texture: Texture) + setParams?(params: unknown) + setClippingPlanes?(planes: Plane[]) +} From 1a44a246ca25ccd6cdf413c355b03f9e1bb92736 Mon Sep 17 00:00:00 2001 From: AlexandruPopovici Date: Fri, 14 Oct 2022 12:48:20 +0300 Subject: [PATCH 08/54] Finished with displaying individual pipeline stages --- packages/viewer-sandbox/src/Sandbox.ts | 172 +++++++----------- .../{DynamicSAOPass.ts => DynamicAOPass.ts} | 55 ++++-- .../viewer/src/modules/pipeline/Pipeline.ts | 77 +++++++- .../src/modules/pipeline/SpecklePass.ts | 1 + 4 files changed, 178 insertions(+), 127 deletions(-) rename packages/viewer/src/modules/pipeline/{DynamicSAOPass.ts => DynamicAOPass.ts} (87%) diff --git a/packages/viewer-sandbox/src/Sandbox.ts b/packages/viewer-sandbox/src/Sandbox.ts index 1dadd4c7e..2718d3074 100644 --- a/packages/viewer-sandbox/src/Sandbox.ts +++ b/packages/viewer-sandbox/src/Sandbox.ts @@ -360,96 +360,74 @@ export default class Sandbox { this.viewer.getRenderer().pipelineOptions = Sandbox.pipelineParams this.viewer.requestRender() }) - // postFolder - // .addInput(Sandbox.postParams, 'saoEnabled', { label: 'SAO-ENABLED' }) - // .on('change', () => { - // this.viewer.getRenderer().pipelineOptions = Sandbox.postParams - // this.viewer.requestRender() - // }) - // postFolder - // .addInput(Sandbox.postParams.saoParams, 'saoBias', { - // min: -1, - // max: 1 - // }) - // .on('change', () => { - // this.viewer.getRenderer().pipelineOptions = Sandbox.postParams - // this.viewer.requestRender() - // }) - // postFolder - // .addInput(Sandbox.postParams.saoParams, 'saoIntensity', { - // min: 0, - // max: 5 - // }) - // .on('change', () => { - // this.viewer.getRenderer().pipelineOptions = Sandbox.postParams - // this.viewer.requestRender() - // }) + const dynamicAoFolder = pipelineFolder.addFolder({ + title: 'Dynamic AO', + expanded: true + }) + dynamicAoFolder + .addInput(Sandbox.pipelineParams.dynamicAoParams, 'intensity', { min: 0, max: 5 }) + .on('change', () => { + this.viewer.getRenderer().pipelineOptions = Sandbox.pipelineParams + this.viewer.requestRender() + }) + dynamicAoFolder + .addInput(Sandbox.pipelineParams.dynamicAoParams, 'kernelRadius', { + min: 0, + max: 500 + }) + .on('change', () => { + this.viewer.getRenderer().pipelineOptions = Sandbox.pipelineParams + this.viewer.requestRender() + }) + dynamicAoFolder + .addInput(Sandbox.pipelineParams.dynamicAoParams, 'bias', { + min: -1, + max: 1 + }) + .on('change', () => { + this.viewer.getRenderer().pipelineOptions = Sandbox.pipelineParams + this.viewer.requestRender() + }) + dynamicAoFolder + .addInput(Sandbox.pipelineParams.dynamicAoParams, 'normalsType', { + options: { + DEFAULT: 0, + ADVANCED: 1, + ACCURATE: 2 + } + }) + .on('change', () => { + this.viewer.getRenderer().pipelineOptions = Sandbox.pipelineParams + this.viewer.requestRender() + }) - // // postFolder - // // .addInput(Sandbox.postParams.saoParams, 'saoScale', { - // // min: 0, - // // max: 100 - // // }) - // // .on('change', () => { - // // this.viewer.getRenderer().pipelineOptions = Sandbox.postParams - // // this.viewer.requestRender() - // // }) - // postFolder - // .addInput(Sandbox.postParams, 'saoScaleOffset', { - // min: -100, - // max: 100 - // }) - // .on('change', () => { - // this.viewer.getRenderer().pipelineOptions = Sandbox.postParams - // this.viewer.requestRender() - // }) + dynamicAoFolder + .addInput(Sandbox.pipelineParams.dynamicAoParams, 'blurEnabled', {}) + .on('change', () => { + this.viewer.getRenderer().pipelineOptions = Sandbox.pipelineParams + this.viewer.requestRender() + }) - // postFolder - // .addInput(Sandbox.postParams, 'saoNormalsRendering', { - // options: { - // DEFAULT: 0, - // ADVANCED: 1, - // ACCURATE: 2 - // } - // }) - // .on('change', () => { - // this.viewer.getRenderer().pipelineOptions = Sandbox.postParams - // this.viewer.requestRender() - // }) + dynamicAoFolder + .addInput(Sandbox.pipelineParams.dynamicAoParams, 'blurRadius', { + min: 0, + max: 10 + }) + .on('change', () => { + this.viewer.getRenderer().pipelineOptions = Sandbox.pipelineParams + this.viewer.requestRender() + }) - // postFolder - // .addInput(Sandbox.postParams.saoParams, 'saoKernelRadius', { - // min: 0, - // max: 100 - // }) - // .on('change', () => { - // this.viewer.getRenderer().pipelineOptions = Sandbox.postParams - // this.viewer.requestRender() - // }) - - // // postFolder - // // .addInput(Sandbox.postParams.saoParams, 'saoMinResolution', { - // // min: 0, - // // max: 1 - // // }) - // // .on('change', () => { - // // this.viewer.getRenderer().pipelineOptions = Sandbox.postParams - // // this.viewer.requestRender() - // // }) - - // postFolder - // .addInput(Sandbox.postParams.saoParams, 'saoBlur', {}) - // .on('change', () => { - // this.viewer.getRenderer().pipelineOptions = Sandbox.postParams - // this.viewer.requestRender() - // }) - - // postFolder - // .addInput(Sandbox.postParams.saoParams, 'saoBlurRadius', { min: 0, max: 10 }) - // .on('change', () => { - // this.viewer.getRenderer().pipelineOptions = Sandbox.postParams - // this.viewer.requestRender() - // }) + dynamicAoFolder + .addInput(Sandbox.pipelineParams.dynamicAoParams, 'blurDepthCutoff', { + min: 0, + max: 1, + step: 0.00001 + }) + .on('change', () => { + this.viewer.getRenderer().pipelineOptions = Sandbox.pipelineParams + this.viewer.requestRender() + }) // postFolder // .addInput(Sandbox.postParams, 'minDistance', { min: 0, max: 100, step: 0.000001 }) @@ -486,26 +464,6 @@ export default class Sandbox { // this.viewer.requestRender() // }) - // postFolder - // .addInput(Sandbox.postParams.saoParams, 'saoBlurStdDev', { - // min: 0, - // max: 150 - // }) - // .on('change', () => { - // this.viewer.getRenderer().pipelineOptions = Sandbox.postParams - // this.viewer.requestRender() - // }) - - // postFolder - // .addInput(Sandbox.postParams.saoParams, 'saoBlurDepthCutoff', { - // min: 0, - // max: 10 - // }) - // .on('change', () => { - // this.viewer.getRenderer().pipelineOptions = Sandbox.postParams - // this.viewer.requestRender() - // }) - const lightsFolder = this.tabs.pages[1].addFolder({ title: 'Lights', expanded: true diff --git a/packages/viewer/src/modules/pipeline/DynamicSAOPass.ts b/packages/viewer/src/modules/pipeline/DynamicAOPass.ts similarity index 87% rename from packages/viewer/src/modules/pipeline/DynamicSAOPass.ts rename to packages/viewer/src/modules/pipeline/DynamicAOPass.ts index d29490af6..68eae83ce 100644 --- a/packages/viewer/src/modules/pipeline/DynamicSAOPass.ts +++ b/packages/viewer/src/modules/pipeline/DynamicAOPass.ts @@ -17,7 +17,11 @@ import { speckleSaoVert } from '../materials/shaders/speckle-sao-vert' import { SAOShader } from 'three/examples/jsm/shaders/SAOShader.js' import { DepthLimitedBlurShader } from 'three/examples/jsm/shaders/DepthLimitedBlurShader.js' import { BlurShaderUtils } from 'three/examples/jsm/shaders/DepthLimitedBlurShader.js' -import { InputDepthTextureUniform, SpecklePass } from './SpecklePass' +import { + InputDepthTextureUniform, + InputNormalsTextureUniform, + SpecklePass +} from './SpecklePass' export enum NormalsType { DEFAULT = 0, @@ -25,7 +29,13 @@ export enum NormalsType { ACCURATE = 2 } -export interface SpeckleDynamicSAOPassParams { +export enum DynamicAOOutputType { + RECONSTRUCTED_NORMALS, + AO, + AO_BLURRED +} + +export interface DynamicAOPassParams { intensity: number scale: number kernelRadius: number @@ -50,7 +60,7 @@ export const DefaultSpeckleDynamicSAOPassParams = { } export class DynamicSAOPass extends Pass implements SpecklePass { - private params: SpeckleDynamicSAOPassParams = DefaultSpeckleDynamicSAOPassParams + private params: DynamicAOPassParams = DefaultSpeckleDynamicSAOPassParams private colorBuffer: Color = new Color() private saoMaterial: ShaderMaterial = null private vBlurMaterial: ShaderMaterial = null @@ -58,6 +68,7 @@ export class DynamicSAOPass extends Pass implements SpecklePass { private saoRenderTarget: WebGLRenderTarget = null private blurIntermediateRenderTarget: WebGLRenderTarget = null private fsQuad: FullScreenQuad = null + private _outputType: DynamicAOOutputType = DynamicAOOutputType.AO_BLURRED private prevStdDev: number private prevNumSamples: number @@ -70,11 +81,6 @@ export class DynamicSAOPass extends Pass implements SpecklePass { return this.saoRenderTarget.texture } - public set outputReconstructedNormals(value: boolean) { - if (value) this.saoMaterial.defines['OUTPUT_RECONSTRUCTED_NORMALS'] = '' - else delete this.saoMaterial.defines['OUTPUT_RECONSTRUCTED_NORMALS'] - } - constructor() { super() @@ -135,16 +141,34 @@ export class DynamicSAOPass extends Pass implements SpecklePass { Object.assign(this.params, params) } - public setTexture(uName: InputDepthTextureUniform, texture: Texture) { - this.saoMaterial.uniforms['tDepth'].value = texture - this.vBlurMaterial.uniforms['tDepth'].value = texture - this.hBlurMaterial.uniforms['tDepth'].value = texture + public setOutputType(type: DynamicAOOutputType) { + this._outputType = type + } + + public setTexture( + uName: InputDepthTextureUniform | InputNormalsTextureUniform, + texture: Texture + ) { + if (uName === 'tDepth') { + this.saoMaterial.uniforms['tDepth'].value = texture + this.vBlurMaterial.uniforms['tDepth'].value = texture + this.hBlurMaterial.uniforms['tDepth'].value = texture + } + if (uName === 'tNormal') { + this.saoMaterial.uniforms['tNormal'].value = texture + } this.saoMaterial.needsUpdate = true this.vBlurMaterial.needsUpdate = true this.hBlurMaterial.needsUpdate = true } public update(scene: Scene, camera: Camera) { + if (this._outputType === DynamicAOOutputType.RECONSTRUCTED_NORMALS) { + this.saoMaterial.defines['OUTPUT_RECONSTRUCTED_NORMALS'] = '' + } else { + delete this.saoMaterial.defines['OUTPUT_RECONSTRUCTED_NORMALS'] + } + this.params.scale = (camera as PerspectiveCamera | OrthographicCamera).far /** SAO DEFINES */ this.saoMaterial.defines['PERSPECTIVE_CAMERA'] = (camera as PerspectiveCamera) @@ -236,8 +260,6 @@ export class DynamicSAOPass extends Pass implements SpecklePass { } public render(renderer) { - const outputNormals = - this.saoMaterial.defines['OUTPUT_RECONSTRUCTED_NORMALS'] !== undefined // Rendering SAO texture renderer.getClearColor(this.colorBuffer) const originalClearAlpha = renderer.getClearAlpha() @@ -253,7 +275,10 @@ export class DynamicSAOPass extends Pass implements SpecklePass { this.fsQuad.material = this.saoMaterial this.fsQuad.render(renderer) - if (this.params.blurEnabled && !outputNormals) { + if ( + this.params.blurEnabled && + this._outputType === DynamicAOOutputType.AO_BLURRED + ) { renderer.setRenderTarget(this.blurIntermediateRenderTarget) renderer.setClearColor(0xffffff) renderer.setClearAlpha(1.0) diff --git a/packages/viewer/src/modules/pipeline/Pipeline.ts b/packages/viewer/src/modules/pipeline/Pipeline.ts index db2dc1a0e..4b4e8c1e9 100644 --- a/packages/viewer/src/modules/pipeline/Pipeline.ts +++ b/packages/viewer/src/modules/pipeline/Pipeline.ts @@ -10,8 +10,10 @@ import { NormalsPass } from './NormalsPass' import { DefaultSpeckleDynamicSAOPassParams, DynamicSAOPass, - SpeckleDynamicSAOPassParams -} from './DynamicSAOPass' + DynamicAOOutputType, + DynamicAOPassParams, + NormalsType +} from './DynamicAOPass' // import { SpecklePass } from './SpecklePass' // import { SpeckleStaticAOGeneratePass } from './SpeckleStaticAOGeneratePass' @@ -30,7 +32,7 @@ export enum PipelineOutputType { export interface PipelineOptions { pipelineOutput: PipelineOutputType dynamicAoEnabled: boolean - dynamicAoParams: SpeckleDynamicSAOPassParams + dynamicAoParams: DynamicAOPassParams } export const DefaultPipelineOptions: PipelineOptions = { @@ -63,12 +65,46 @@ export class Pipeline { public set pipelineOptions(options: Partial) { Object.assign(this._pipelineOptions, options) - this.pipelineOutput = options.pipelineOutput + if (options.dynamicAoEnabled) { + this.dynamicAoPass.enabled = true + this.renderPass.enabled = true + this.applySaoPass.enabled = true + this.normalsPass.enabled = + options.dynamicAoParams.normalsType === NormalsType.DEFAULT ? true : false + this.depthPass.enabled = true + this.copyOutputPass.enabled = false + } else { + this.depthPass.enabled = false + this.dynamicAoPass.enabled = false + this.applySaoPass.enabled = false + this.copyOutputPass.enabled = false + this.normalsPass.enabled = false + this.renderPass.enabled = true + } this.dynamicAoPass.setParams(options.dynamicAoParams) + + this.pipelineOutput = options.pipelineOutput } public set pipelineOutput(outputType: PipelineOutputType) { switch (outputType) { + case PipelineOutputType.FINAL: + this.dynamicAoPass.enabled = true + this.renderPass.enabled = true + this.applySaoPass.enabled = true + this.normalsPass.enabled = + this._pipelineOptions.dynamicAoParams.normalsType === NormalsType.DEFAULT + ? true + : false + this.depthPass.enabled = true + this.copyOutputPass.enabled = false + this.dynamicAoPass.setOutputType( + this._pipelineOptions.dynamicAoParams.blurEnabled + ? DynamicAOOutputType.AO_BLURRED + : DynamicAOOutputType.AO + ) + break + case PipelineOutputType.DEPTH_RGBA: this.dynamicAoPass.enabled = false this.renderPass.enabled = false @@ -120,7 +156,37 @@ export class Pipeline { this.copyOutputPass.enabled = true this.copyOutputPass.setTexture('tDiffuse', this.dynamicAoPass.outputTexture) this.copyOutputPass.setOutputType(PipelineOutputType.GEOMETRY_NORMALS) - this.dynamicAoPass.outputReconstructedNormals = true + this.dynamicAoPass.setOutputType(DynamicAOOutputType.RECONSTRUCTED_NORMALS) + break + + case PipelineOutputType.DYNAMIC_AO: + this.depthPass.enabled = true + this.dynamicAoPass.enabled = true + this.applySaoPass.enabled = false + this.renderPass.enabled = false + this.normalsPass.enabled = + this._pipelineOptions.dynamicAoParams.normalsType === NormalsType.DEFAULT + ? true + : false + this.copyOutputPass.enabled = true + this.copyOutputPass.setTexture('tDiffuse', this.dynamicAoPass.outputTexture) + this.copyOutputPass.setOutputType(PipelineOutputType.COLOR) + this.dynamicAoPass.setOutputType(DynamicAOOutputType.AO) + break + + case PipelineOutputType.DYNAMIC_AO_BLURED: + this.depthPass.enabled = true + this.dynamicAoPass.enabled = true + this.applySaoPass.enabled = false + this.renderPass.enabled = false + this.normalsPass.enabled = + this._pipelineOptions.dynamicAoParams.normalsType === NormalsType.DEFAULT + ? true + : false + this.copyOutputPass.enabled = true + this.copyOutputPass.setTexture('tDiffuse', this.dynamicAoPass.outputTexture) + this.copyOutputPass.setOutputType(PipelineOutputType.COLOR) + this.dynamicAoPass.setOutputType(DynamicAOOutputType.AO_BLURRED) break default: break @@ -155,6 +221,7 @@ export class Pipeline { this.composer.addPass(this.copyOutputPass) this.dynamicAoPass.setTexture('tDepth', this.depthPass.outputTexture) + this.dynamicAoPass.setTexture('tNormal', this.normalsPass.outputTexture) this.applySaoPass.setTexture('tDiffuse', this.dynamicAoPass.outputTexture) let restoreVisibility diff --git a/packages/viewer/src/modules/pipeline/SpecklePass.ts b/packages/viewer/src/modules/pipeline/SpecklePass.ts index 0451fcb50..bc71e494e 100644 --- a/packages/viewer/src/modules/pipeline/SpecklePass.ts +++ b/packages/viewer/src/modules/pipeline/SpecklePass.ts @@ -2,6 +2,7 @@ import { Camera, Plane, Scene, Texture } from 'three' export type InputColorTextureUniform = 'tDiffuse' export type InputDepthTextureUniform = 'tDepth' +export type InputNormalsTextureUniform = 'tNormal' export interface SpecklePass { onBeforeRender?: () => void From 5d895a96ff86451fc19d168366d06522a28e714e Mon Sep 17 00:00:00 2001 From: AlexandruPopovici Date: Fri, 14 Oct 2022 19:26:26 +0300 Subject: [PATCH 09/54] Intruduced accumulation frames properly at pipeline level. Cleaned up progressive AO shader and pass. Exposed the number of accumulation frames as well as the kernel size in the sandbox. --- packages/viewer-sandbox/src/Sandbox.ts | 101 +++++++++---- packages/viewer-sandbox/src/main.ts | 4 +- .../speckle-static-ao-accumulate-frag.ts | 4 +- .../speckle-static-ao-generate-frag.ts | 31 +--- .../src/modules/pipeline/DynamicAOPass.ts | 4 +- .../viewer/src/modules/pipeline/Pipeline.ts | 89 +++++++++--- .../src/modules/pipeline/SpecklePass.ts | 4 + ...taticAOGeneratePass.ts => StaticAOPass.ts} | 137 +++++++++++------- 8 files changed, 242 insertions(+), 132 deletions(-) rename packages/viewer/src/modules/pipeline/{SpeckleStaticAOGeneratePass.ts => StaticAOPass.ts} (73%) diff --git a/packages/viewer-sandbox/src/Sandbox.ts b/packages/viewer-sandbox/src/Sandbox.ts index 2718d3074..4a26bbc0e 100644 --- a/packages/viewer-sandbox/src/Sandbox.ts +++ b/packages/viewer-sandbox/src/Sandbox.ts @@ -32,6 +32,7 @@ export default class Sandbox { public static pipelineParams = { pipelineOutput: 8, + accumulationFrames: 16, dynamicAoEnabled: true, dynamicAoParams: { intensity: 1.25, @@ -43,6 +44,14 @@ export default class Sandbox { blurRadius: 4, blurStdDev: 4, blurDepthCutoff: 0.0007 + }, + staticAoEnabled: true, + staticAoParams: { + intensity: 1, + kernelRadius: 0.5, // World space + kernelSize: 16, + minDistance: 0, + maxDistance: 0.008 } } @@ -360,16 +369,30 @@ export default class Sandbox { this.viewer.getRenderer().pipelineOptions = Sandbox.pipelineParams this.viewer.requestRender() }) + + pipelineFolder + .addInput(Sandbox.pipelineParams, 'accumulationFrames', { + min: 1, + max: 128, + step: 1 + }) + .on('change', () => { + this.viewer.getRenderer().pipelineOptions = Sandbox.pipelineParams + this.viewer.requestRender() + }) + const dynamicAoFolder = pipelineFolder.addFolder({ title: 'Dynamic AO', expanded: true }) + dynamicAoFolder .addInput(Sandbox.pipelineParams.dynamicAoParams, 'intensity', { min: 0, max: 5 }) .on('change', () => { this.viewer.getRenderer().pipelineOptions = Sandbox.pipelineParams this.viewer.requestRender() }) + dynamicAoFolder .addInput(Sandbox.pipelineParams.dynamicAoParams, 'kernelRadius', { min: 0, @@ -379,6 +402,7 @@ export default class Sandbox { this.viewer.getRenderer().pipelineOptions = Sandbox.pipelineParams this.viewer.requestRender() }) + dynamicAoFolder .addInput(Sandbox.pipelineParams.dynamicAoParams, 'bias', { min: -1, @@ -429,40 +453,57 @@ export default class Sandbox { this.viewer.requestRender() }) - // postFolder - // .addInput(Sandbox.postParams, 'minDistance', { min: 0, max: 100, step: 0.000001 }) + const staticAoFolder = pipelineFolder.addFolder({ + title: 'Static AO', + expanded: true + }) + // staticAoFolder + // .addInput(Sandbox.pipelineParams, 'staticAoEnabled', {}) // .on('change', () => { - // this.viewer.getRenderer().pipelineOptions = Sandbox.postParams + // this.viewer.getRenderer().pipelineOptions = Sandbox.pipelineParams // this.viewer.requestRender() // }) + staticAoFolder + .addInput(Sandbox.pipelineParams.staticAoParams, 'minDistance', { + min: 0, + max: 100, + step: 0.000001 + }) + .on('change', () => { + this.viewer.getRenderer().pipelineOptions = Sandbox.pipelineParams + this.viewer.requestRender() + }) - // postFolder - // .addInput(Sandbox.postParams, 'maxDistance', { min: 0, max: 100, step: 0.000001 }) - // .on('change', () => { - // this.viewer.getRenderer().pipelineOptions = Sandbox.postParams - // this.viewer.requestRender() - // }) - // postFolder - // .addInput(Sandbox.postParams, 'ssaoKernelRadius', { min: 0, max: 100 }) - // .on('change', () => { - // this.viewer.getRenderer().pipelineOptions = Sandbox.postParams - // this.viewer.requestRender() - // }) - // postFolder.addInput(Sandbox.postParams, 'progressive', {}).on('change', () => { - // this.viewer.getRenderer().pipelineOptions = Sandbox.postParams - // this.viewer.requestRender() - // }) - // postFolder - // .addInput(Sandbox.postParams, 'progressiveAO', { - // options: { - // SAO: 0, - // SSAO: 1 - // } - // }) - // .on('change', () => { - // this.viewer.getRenderer().pipelineOptions = Sandbox.postParams - // this.viewer.requestRender() - // }) + staticAoFolder + .addInput(Sandbox.pipelineParams.staticAoParams, 'maxDistance', { + min: 0, + max: 100, + step: 0.000001 + }) + .on('change', () => { + this.viewer.getRenderer().pipelineOptions = Sandbox.pipelineParams + this.viewer.requestRender() + }) + staticAoFolder + .addInput(Sandbox.pipelineParams.staticAoParams, 'kernelRadius', { + min: 0, + max: 100 + }) + .on('change', () => { + this.viewer.getRenderer().pipelineOptions = Sandbox.pipelineParams + this.viewer.requestRender() + }) + + staticAoFolder + .addInput(Sandbox.pipelineParams.staticAoParams, 'kernelSize', { + min: 1, + max: 128, + step: 1 + }) + .on('change', () => { + this.viewer.getRenderer().pipelineOptions = Sandbox.pipelineParams + this.viewer.requestRender() + }) const lightsFolder = this.tabs.pages[1].addFolder({ title: 'Lights', diff --git a/packages/viewer-sandbox/src/main.ts b/packages/viewer-sandbox/src/main.ts index 067770572..1a1b0dcb8 100644 --- a/packages/viewer-sandbox/src/main.ts +++ b/packages/viewer-sandbox/src/main.ts @@ -80,9 +80,9 @@ sandbox.makeFilteringUI() // Load demo object await sandbox.loadUrl( - // 'https://speckle.xyz/streams/da9e320dad/commits/5388ef24b8?c=%5B-7.66134,10.82932,6.41935,-0.07739,-13.88552,1.8697,0,1%5D' + 'https://speckle.xyz/streams/da9e320dad/commits/5388ef24b8?c=%5B-7.66134,10.82932,6.41935,-0.07739,-13.88552,1.8697,0,1%5D' // Revit sample house (good for bim-like stuff with many display meshes) - 'https://speckle.xyz/streams/da9e320dad/commits/5388ef24b8' + // 'https://speckle.xyz/streams/da9e320dad/commits/5388ef24b8' // 'Super' heavy revit shit // 'https://speckle.xyz/streams/e6f9156405/commits/0694d53bb5' // Same sample revit house, local to dim's computer diff --git a/packages/viewer/src/modules/materials/shaders/speckle-static-ao-accumulate-frag.ts b/packages/viewer/src/modules/materials/shaders/speckle-static-ao-accumulate-frag.ts index 8b4298930..728582776 100644 --- a/packages/viewer/src/modules/materials/shaders/speckle-static-ao-accumulate-frag.ts +++ b/packages/viewer/src/modules/materials/shaders/speckle-static-ao-accumulate-frag.ts @@ -2,10 +2,10 @@ export const speckleStaticAoAccumulateFrag = /* glsl */ ` uniform float opacity; uniform sampler2D tDiffuse; varying vec2 vUv; - #define NUM_FRAMES 16 + // #define NUM_FRAMES 16 void main() { vec4 frameSample = texture2D( tDiffuse, vUv ); - gl_FragColor.xyz = (frameSample.rgb) * 1./float(NUM_FRAMES); + gl_FragColor.xyz = frameSample.rgb * 1./float(NUM_FRAMES); gl_FragColor.a = 1.;//*= opacity; }` diff --git a/packages/viewer/src/modules/materials/shaders/speckle-static-ao-generate-frag.ts b/packages/viewer/src/modules/materials/shaders/speckle-static-ao-generate-frag.ts index 8bfdd11ac..af007d73b 100644 --- a/packages/viewer/src/modules/materials/shaders/speckle-static-ao-generate-frag.ts +++ b/packages/viewer/src/modules/materials/shaders/speckle-static-ao-generate-frag.ts @@ -17,16 +17,17 @@ export const speckleStaticAoGenerateFrag = /* glsl */ ` uniform float minResolution; uniform float frameIndex; - #define KERNEL_SIZE 16 + #define AO_ESTIMATOR 1 + // #define KERNEL_SIZE 16 uniform sampler2D tNoise; uniform vec3 kernel[ KERNEL_SIZE ]; uniform float minDistance; uniform float maxDistance; - uniform float ssaoKernelRadius; #define NUM_SAMPLES 16 #define SPIRAL_TURNS 2 - #define NUM_FRAMES 16 + + // #define NUM_FRAMES 16 #define NORMAL_TEXTURE 0 #define IMPROVED_NORMAL_RECONSTRUCTION 0 @@ -210,35 +211,19 @@ export const speckleStaticAoGenerateFrag = /* glsl */ ` float sampleViewZ = getViewZ( sampleDepth ); vec3 sampleViewPosition = getViewPosition( sampleUv, sampleDepth, sampleViewZ ); + /** McGuire Estimator*/ vec3 v = sampleViewPosition - centerViewPosition; - float vv = dot(v, v); - float vn = dot(v, centerViewNormal);// - uBias; + float vn = dot(v, centerViewNormal) - bias; - // #if VARIATION == 0 - - // (from the HPG12 paper) // Note large epsilon to avoid overdarkening within cracks float radius2 = 2.;//uSampleRadiusWS * uSampleRadiusWS float epsilon = 0.01; - // occlusionSum += float(vv < radius2) * max(vn / (epsilon + vv), 0.0) / 4.; - // #elif VARIATION == 1 // default / recommended - - // Smoother transition to zero (lowers contrast, smoothing out corners). [Recommended] float f = max(radius2 - vv, 0.0) / radius2; occlusionSum += f * f * f * max(vn / (epsilon + vv), 0.0) / 4.; - - // #elif VARIATION == 2 - - // // Medium contrast (which looks better at high radii), no division. Note that the - // // contribution still falls off with radius^2, but we've adjusted the rate in a way that is - // // more computationally efficient and happens to be aesthetically pleasing. - // float invRadius2 = 1.0 / radius2; - // return 4.0 * max(1.0 - vv * invRadius2, 0.0) * max(vn, 0.0); - - + /** Three.js SAO Estimator*/ // vec3 viewDelta = sampleViewPosition - centerViewPosition; // float viewDistance = length( viewDelta ); // float scaledScreenDistance = scaleDividedByCameraFar * viewDistance; @@ -259,7 +244,7 @@ export const speckleStaticAoGenerateFrag = /* glsl */ ` float occlusion = 0.0; for ( int i = 0; i < KERNEL_SIZE; i ++ ) { vec3 sampleVector = kernelMatrix * kernel[ i ]; // reorient sample vector in view space - vec3 samplePoint = viewPosition + ( sampleVector * ssaoKernelRadius ); // calculate sample point + vec3 samplePoint = viewPosition + ( sampleVector * kernelRadius ); // calculate sample point vec4 samplePointNDC = cameraProjectionMatrix * vec4( samplePoint, 1.0 ); // project point and calculate NDC samplePointNDC /= samplePointNDC.w; vec2 samplePointUv = samplePointNDC.xy * 0.5 + 0.5; // compute uv coordinates diff --git a/packages/viewer/src/modules/pipeline/DynamicAOPass.ts b/packages/viewer/src/modules/pipeline/DynamicAOPass.ts index 68eae83ce..a1da743ea 100644 --- a/packages/viewer/src/modules/pipeline/DynamicAOPass.ts +++ b/packages/viewer/src/modules/pipeline/DynamicAOPass.ts @@ -47,7 +47,7 @@ export interface DynamicAOPassParams { blurDepthCutoff: number } -export const DefaultSpeckleDynamicSAOPassParams = { +export const DefaultDynamicAOPassParams = { intensity: 1.25, scale: 0, kernelRadius: 10, @@ -60,7 +60,7 @@ export const DefaultSpeckleDynamicSAOPassParams = { } export class DynamicSAOPass extends Pass implements SpecklePass { - private params: DynamicAOPassParams = DefaultSpeckleDynamicSAOPassParams + private params: DynamicAOPassParams = DefaultDynamicAOPassParams private colorBuffer: Color = new Color() private saoMaterial: ShaderMaterial = null private vBlurMaterial: ShaderMaterial = null diff --git a/packages/viewer/src/modules/pipeline/Pipeline.ts b/packages/viewer/src/modules/pipeline/Pipeline.ts index 4b4e8c1e9..8257944e4 100644 --- a/packages/viewer/src/modules/pipeline/Pipeline.ts +++ b/packages/viewer/src/modules/pipeline/Pipeline.ts @@ -8,14 +8,22 @@ import { CopyOutputPass } from './CopyOutputPass' import { DepthPass } from './DepthPass' import { NormalsPass } from './NormalsPass' import { - DefaultSpeckleDynamicSAOPassParams, + DefaultDynamicAOPassParams, DynamicSAOPass, DynamicAOOutputType, DynamicAOPassParams, NormalsType } from './DynamicAOPass' -// import { SpecklePass } from './SpecklePass' -// import { SpeckleStaticAOGeneratePass } from './SpeckleStaticAOGeneratePass' +import { + DefaultStaticAoPassParams, + StaticAOPass, + StaticAoPassParams +} from './StaticAOPass' + +export enum RenderType { + NORMAL, + ACCUMULATION +} export enum PipelineOutputType { DEPTH_RGBA = 0, @@ -31,24 +39,25 @@ export enum PipelineOutputType { export interface PipelineOptions { pipelineOutput: PipelineOutputType + accumulationFrames: number dynamicAoEnabled: boolean dynamicAoParams: DynamicAOPassParams + staticAoEnabled: boolean + staticAoParams: StaticAoPassParams } export const DefaultPipelineOptions: PipelineOptions = { pipelineOutput: PipelineOutputType.FINAL, + accumulationFrames: 16, dynamicAoEnabled: true, - dynamicAoParams: DefaultSpeckleDynamicSAOPassParams - // saoScaleOffset: 0, - // saoNormalsRendering: NormalsType.ACCURATE, - // minDistance: 0, - // maxDistance: 0.008, - // ssaoKernelRadius: 0.5, - // progressiveAO: 0, - // progressive: true + dynamicAoParams: DefaultDynamicAOPassParams, + staticAoEnabled: true, + staticAoParams: DefaultStaticAoPassParams } export class Pipeline { + public static ACCUMULATE_FRAMES = 16 + private _renderer: WebGLRenderer = null private _batcher: Batcher = null private _pipelineOptions: PipelineOptions = Object.assign({}, DefaultPipelineOptions) @@ -61,7 +70,11 @@ export class Pipeline { private applySaoPass: ApplySAOPass = null private copyOutputPass: CopyOutputPass = null + private staticAoPass: StaticAOPass = null + private drawingSize: Vector2 = new Vector2() + private renderType: RenderType = RenderType.NORMAL + private accumulationFrame = 0 public set pipelineOptions(options: Partial) { Object.assign(this._pipelineOptions, options) @@ -82,6 +95,9 @@ export class Pipeline { this.renderPass.enabled = true } this.dynamicAoPass.setParams(options.dynamicAoParams) + this.staticAoPass.setParams(options.staticAoParams) + this.accumulationFrame = 0 + Pipeline.ACCUMULATE_FRAMES = options.accumulationFrames this.pipelineOutput = options.pipelineOutput } @@ -188,6 +204,19 @@ export class Pipeline { this.copyOutputPass.setOutputType(PipelineOutputType.COLOR) this.dynamicAoPass.setOutputType(DynamicAOOutputType.AO_BLURRED) break + + case PipelineOutputType.PROGRESSIVE_AO: + this.depthPass.enabled = true + this.normalsPass.enabled = false + this.dynamicAoPass.enabled = false + this.renderPass.enabled = false + this.applySaoPass.enabled = false + this.staticAoPass.enabled = true + this.copyOutputPass.enabled = true + this.applySaoPass.setTexture('tDiffuse', this.staticAoPass.outputTexture) + this.copyOutputPass.setTexture('tDiffuse', this.staticAoPass.outputTexture) + this.copyOutputPass.setOutputType(PipelineOutputType.COLOR) + break default: break } @@ -210,12 +239,17 @@ export class Pipeline { this.renderPass.renderToScreen = true this.applySaoPass = new ApplySAOPass() this.applySaoPass.renderToScreen = true + + this.staticAoPass = new StaticAOPass() + this.staticAoPass.enabled = false + this.copyOutputPass = new CopyOutputPass() this.copyOutputPass.renderToScreen = true this.copyOutputPass.enabled = false this.composer.addPass(this.depthPass) this.composer.addPass(this.normalsPass) this.composer.addPass(this.dynamicAoPass) + this.composer.addPass(this.staticAoPass) this.composer.addPass(this.renderPass) this.composer.addPass(this.applySaoPass) this.composer.addPass(this.copyOutputPass) @@ -223,6 +257,7 @@ export class Pipeline { this.dynamicAoPass.setTexture('tDepth', this.depthPass.outputTexture) this.dynamicAoPass.setTexture('tNormal', this.normalsPass.outputTexture) this.applySaoPass.setTexture('tDiffuse', this.dynamicAoPass.outputTexture) + this.staticAoPass.setTexture('tDepth', this.depthPass.outputTexture) let restoreVisibility this.depthPass.onBeforeRender = () => { @@ -252,6 +287,8 @@ export class Pipeline { this.depthPass.update(renderer.scene, renderer.camera) this.dynamicAoPass.update(renderer.scene, renderer.camera) this.normalsPass.update(renderer.scene, renderer.camera) + this.staticAoPass.update(renderer.scene, renderer.camera) + this.staticAoPass.setFrameIndex(this.accumulationFrame) } public render(): boolean { @@ -259,8 +296,15 @@ export class Pipeline { if (this.drawingSize.length() === 0) return this._renderer.clear(true) - this.composer.render() - return true + if (this.renderType === RenderType.NORMAL) { + this.composer.render() + return true + } else { + console.warn('Rendering accumulation frame -> ', this.accumulationFrame) + this.composer.render() + this.accumulationFrame++ + return this.accumulationFrame < Pipeline.ACCUMULATE_FRAMES + } } public resize(width: number, height: number) { @@ -268,15 +312,24 @@ export class Pipeline { } public onStationaryBegin() { - // this.renderType = RenderType.ACCUMULATION - // this.staticAOGenerationPass.enabled = true - // this.accumulationFrame = 0 + this.renderType = RenderType.ACCUMULATION + this.accumulationFrame = 0 + this.depthPass.enabled = true + this.normalsPass.enabled = false + this.dynamicAoPass.enabled = false + this.renderPass.enabled = true + this.applySaoPass.enabled = true + this.staticAoPass.enabled = true + this.applySaoPass.setTexture('tDiffuse', this.staticAoPass.outputTexture) console.warn('Starting stationary') } public onStationaryEnd() { - // this.renderType = RenderType.NORMAL - // this.staticAOGenerationPass.enabled = false + this.renderType = RenderType.NORMAL + this.staticAoPass.enabled = false + this.applySaoPass.enabled = true + this.dynamicAoPass.enabled = true + this.applySaoPass.setTexture('tDiffuse', this.dynamicAoPass.outputTexture) console.warn('Ending stationary') } } diff --git a/packages/viewer/src/modules/pipeline/SpecklePass.ts b/packages/viewer/src/modules/pipeline/SpecklePass.ts index bc71e494e..7dc2b2c9b 100644 --- a/packages/viewer/src/modules/pipeline/SpecklePass.ts +++ b/packages/viewer/src/modules/pipeline/SpecklePass.ts @@ -16,3 +16,7 @@ export interface SpecklePass { setParams?(params: unknown) setClippingPlanes?(planes: Plane[]) } + +export interface SpeckleProgressivePass extends SpecklePass { + setFrameIndex(index: number) +} diff --git a/packages/viewer/src/modules/pipeline/SpeckleStaticAOGeneratePass.ts b/packages/viewer/src/modules/pipeline/StaticAOPass.ts similarity index 73% rename from packages/viewer/src/modules/pipeline/SpeckleStaticAOGeneratePass.ts rename to packages/viewer/src/modules/pipeline/StaticAOPass.ts index 315ed41b0..96eace04d 100644 --- a/packages/viewer/src/modules/pipeline/SpeckleStaticAOGeneratePass.ts +++ b/packages/viewer/src/modules/pipeline/StaticAOPass.ts @@ -14,6 +14,7 @@ import { RedFormat, RepeatWrapping, ReverseSubtractEquation, + Scene, ShaderMaterial, Texture, Vector2, @@ -23,61 +24,76 @@ import { } from 'three' import { FullScreenQuad, Pass } from 'three/examples/jsm/postprocessing/Pass' -import Batcher from '../batching/Batcher' import { speckleStaticAoGenerateVert } from '../materials/shaders/speckle-static-ao-generate-vert' import { speckleStaticAoGenerateFrag } from '../materials/shaders/speckle-static-ao-generate-frag' import { speckleStaticAoAccumulateVert } from '../materials/shaders/speckle-static-ao-accumulate-vert' import { speckleStaticAoAccumulateFrag } from '../materials/shaders/speckle-static-ao-accumulate-frag' import { SimplexNoise } from 'three/examples/jsm//math/SimplexNoise.js' +import { + InputDepthTextureUniform, + InputNormalsTextureUniform, + SpeckleProgressivePass +} from './SpecklePass' +import { Pipeline } from './Pipeline' /** * SAO implementation inspired from bhouston previous SAO work */ -export class SpeckleStaticAOGeneratePass extends Pass { - private batcher: Batcher = null +export interface StaticAoPassParams { + intensity: number + kernelRadius: number + kernelSize: number + minDistance: number + maxDistance: number +} + +export const DefaultStaticAoPassParams = { + intensity: 1, + kernelRadius: 0.5, // World space + kernelSize: 16, + minDistance: 0, + maxDistance: 0.008 +} + +export class StaticAOPass extends Pass implements SpeckleProgressivePass { public aoMaterial: ShaderMaterial = null private accumulateMaterial: ShaderMaterial = null - private _depthTexture: Texture - private _normalTexture: Texture private _generationBuffer: WebGLRenderTarget private _accumulationBuffer: WebGLRenderTarget + private params: StaticAoPassParams = DefaultStaticAoPassParams private fsQuad: FullScreenQuad private frameIndex = 0 - private kernels: Array> = new Array(16) - private kernelSize = 16 + private kernels: Array> = [] private noiseTextures: Array = [] - public minDistance = 0.02 - public maxDistance = 0.3 - public ssaoKernelRadius = 1 - public progressiveAO = 0 - public kernelRadius = 15 - public set depthTexture(value: Texture) { - this._depthTexture = value - this.aoMaterial.uniforms['tDepth'].value = value + public setTexture( + uName: InputDepthTextureUniform | InputNormalsTextureUniform, + texture: Texture + ) { + if (uName === 'tDepth') { + this.aoMaterial.uniforms['tDepth'].value = texture + } + if (uName === 'tNormal') { + this.aoMaterial.uniforms['tNormal'].value = texture + } this.aoMaterial.needsUpdate = true } - public set normalTexture(value: Texture) { - this._normalTexture = value - } - public get outputTexture() { - return this._accumulationBuffer + return this._accumulationBuffer.texture } - constructor(batcher: Batcher) { + public get displayName(): string { + return 'STATIC-AO' + } + + constructor() { super() - this.batcher = batcher this._generationBuffer = new WebGLRenderTarget(256, 256) this._accumulationBuffer = new WebGLRenderTarget(256, 256) - const aoDefines = { - AO_ESTIMATOR: 0 - } this.aoMaterial = new ShaderMaterial({ - defines: aoDefines, fragmentShader: speckleStaticAoGenerateFrag, vertexShader: speckleStaticAoGenerateVert, uniforms: { @@ -95,16 +111,14 @@ export class SpeckleStaticAOGeneratePass extends Pass { bias: { value: 0 }, minResolution: { value: 0.0 }, - kernelRadius: { value: 15.0 }, - randomSeed: { value: 0.0 }, + kernelRadius: { value: 0.5 }, // World space frameIndex: { value: 0 }, tNoise: { value: null }, kernel: { value: null }, minDistance: { value: 0.0 }, - maxDistance: { value: 0.008 }, - ssaoKernelRadius: { value: 0.5 } + maxDistance: { value: 1 } } }) @@ -133,7 +147,26 @@ export class SpeckleStaticAOGeneratePass extends Pass { this.fsQuad = new FullScreenQuad(this.aoMaterial) } - public update(camera: Camera, frameIndex: number) { + public setParams(params: unknown) { + Object.assign(this.params, params) + this.kernels = [] + this.noiseTextures = [] + } + + public setFrameIndex(index: number) { + this.frameIndex = index + } + + public update(scene: Scene, camera: Camera) { + /** DEFINES */ + this.aoMaterial.defines['PERSPECTIVE_CAMERA'] = (camera as PerspectiveCamera) + .isPerspectiveCamera + ? 1 + : 0 + this.aoMaterial.defines['NUM_FRAMES'] = Pipeline.ACCUMULATE_FRAMES + this.aoMaterial.defines['KERNEL_SIZE'] = this.params.kernelSize + this.accumulateMaterial.defines['NUM_FRAMES'] = Pipeline.ACCUMULATE_FRAMES + /** UNIFORMS */ this.aoMaterial.uniforms['cameraNear'].value = ( camera as PerspectiveCamera | OrthographicCamera ).near @@ -146,29 +179,22 @@ export class SpeckleStaticAOGeneratePass extends Pass { this.aoMaterial.uniforms['cameraProjectionMatrix'].value.copy( camera.projectionMatrix ) - this.aoMaterial.uniforms['scale'].value = ( - camera as PerspectiveCamera | OrthographicCamera - ).far - this.aoMaterial.defines['PERSPECTIVE_CAMERA'] = (camera as PerspectiveCamera) - .isPerspectiveCamera - ? 1 - : 0 - this.aoMaterial.uniforms['kernelRadius'].value = this.kernelRadius - this.aoMaterial.uniforms['frameIndex'].value = frameIndex - this.frameIndex = frameIndex + if (!this.kernels[this.frameIndex]) { + this.generateSampleKernel(this.frameIndex) + } + if (!this.noiseTextures[this.frameIndex]) { + this.generateRandomKernelRotations(this.frameIndex) + } + this.aoMaterial.uniforms['kernel'].value = this.kernels[this.frameIndex] + this.aoMaterial.uniforms['tNoise'].value = this.noiseTextures[this.frameIndex] - if (!this.kernels[frameIndex]) { - this.generateSampleKernel(frameIndex) - } - if (!this.noiseTextures[frameIndex]) { - this.generateRandomKernelRotations(frameIndex) - } - this.aoMaterial.uniforms['kernel'].value = this.kernels[frameIndex] - this.aoMaterial.uniforms['tNoise'].value = this.noiseTextures[frameIndex] - this.aoMaterial.uniforms['ssaoKernelRadius'].value = this.ssaoKernelRadius - this.aoMaterial.uniforms['minDistance'].value = this.minDistance - this.aoMaterial.uniforms['maxDistance'].value = this.maxDistance + this.aoMaterial.uniforms['kernelRadius'].value = this.params.kernelRadius + this.aoMaterial.uniforms['frameIndex'].value = this.frameIndex + + this.aoMaterial.uniforms['minDistance'].value = this.params.minDistance + this.aoMaterial.uniforms['maxDistance'].value = this.params.maxDistance this.aoMaterial.needsUpdate = true + this.accumulateMaterial.needsUpdate = true } public render(renderer, writeBuffer, readBuffer) { @@ -192,7 +218,7 @@ export class SpeckleStaticAOGeneratePass extends Pass { renderer.autoClear = false renderer.setClearColor(0x000000) renderer.setClearAlpha(1) - renderer.clear() + renderer.clear(true) this.fsQuad.material = this.aoMaterial this.fsQuad.render(renderer) @@ -200,8 +226,9 @@ export class SpeckleStaticAOGeneratePass extends Pass { if (this.frameIndex === 0) { renderer.setClearColor(0xffffff) renderer.setClearAlpha(1) - renderer.clear() + renderer.clear(true) } + this.fsQuad.material = this.accumulateMaterial this.fsQuad.render(renderer) } @@ -215,7 +242,7 @@ export class SpeckleStaticAOGeneratePass extends Pass { } private generateSampleKernel(frameIndex: number) { - const kernelSize = this.kernelSize + const kernelSize = this.params.kernelSize this.kernels[frameIndex] = [] for (let i = 0; i < kernelSize; i++) { From fe79c04f0ff705b29ca02636ebc8f7e3c73ce730 Mon Sep 17 00:00:00 2001 From: AlexandruPopovici Date: Fri, 14 Oct 2022 19:28:15 +0300 Subject: [PATCH 10/54] Accumulation frame count is now reset after a resize --- packages/viewer/src/modules/pipeline/Pipeline.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/viewer/src/modules/pipeline/Pipeline.ts b/packages/viewer/src/modules/pipeline/Pipeline.ts index 8257944e4..02ea0fe40 100644 --- a/packages/viewer/src/modules/pipeline/Pipeline.ts +++ b/packages/viewer/src/modules/pipeline/Pipeline.ts @@ -309,6 +309,7 @@ export class Pipeline { public resize(width: number, height: number) { this.composer.setSize(width, height) + this.accumulationFrame = 0 } public onStationaryBegin() { From 2a8ad8456352cc6f681cce83527678ab50bc7b55 Mon Sep 17 00:00:00 2001 From: AlexandruPopovici Date: Mon, 17 Oct 2022 11:33:35 +0300 Subject: [PATCH 11/54] In the end I just implemented my own stationary/dynamic detection routine since the events sent from the 'camera controller' are dogshit --- packages/viewer/src/modules/DebugViewer.ts | 2 +- .../viewer/src/modules/SpeckleRenderer.ts | 86 +++++++++---------- packages/viewer/src/modules/Viewer.ts | 7 +- .../viewer/src/modules/pipeline/Pipeline.ts | 5 ++ 4 files changed, 46 insertions(+), 54 deletions(-) diff --git a/packages/viewer/src/modules/DebugViewer.ts b/packages/viewer/src/modules/DebugViewer.ts index d449d3f91..eff6b8055 100644 --- a/packages/viewer/src/modules/DebugViewer.ts +++ b/packages/viewer/src/modules/DebugViewer.ts @@ -5,7 +5,7 @@ export class DebugViewer extends Viewer { return this.speckleRenderer } requestRender() { - this.needsRender = true + this.speckleRenderer.needsRender = true } requestRenderShadowmap() { this.getRenderer().updateDirectLights() diff --git a/packages/viewer/src/modules/SpeckleRenderer.ts b/packages/viewer/src/modules/SpeckleRenderer.ts index 0f01cd9e8..5611cbc0b 100644 --- a/packages/viewer/src/modules/SpeckleRenderer.ts +++ b/packages/viewer/src/modules/SpeckleRenderer.ts @@ -2,7 +2,6 @@ import { ACESFilmicToneMapping, Box3, Box3Helper, - Camera, CameraHelper, Color, DirectionalLight, @@ -51,6 +50,7 @@ export default class SpeckleRenderer { private readonly SHOW_HELPERS = false private _renderer: WebGLRenderer public _scene: Scene + private _needsRender: boolean private rootGroup: Group private batcher: Batcher private intersections: Intersections @@ -61,11 +61,19 @@ export default class SpeckleRenderer { public viewer: Viewer // TEMPORARY private filterBatchRecording: string[] private pipeline: Pipeline + private lastAzimuth: number + private lastPolar: number + private lastCameraPosition: Vector3 = new Vector3() + private lastCameraMotionDelta: number public get renderer(): WebGLRenderer { return this._renderer } + public set needsRender(value: boolean) { + this._needsRender ||= value + } + public set indirectIBL(texture: Texture) { this._scene.environment = texture } @@ -178,19 +186,10 @@ export default class SpeckleRenderer { camHelper.name = 'CamHelper' helpers.add(camHelper) } - this.viewer.cameraHandler.controls.restThreshold = 0.001 - this.viewer.cameraHandler.controls.addEventListener('sleep', () => { - this.pipeline.onStationaryBegin() - }) - // this.viewer.cameraHandler.controls.addEventListener('rest', (event) => { - // this.pipeline.onStationaryBegin() - // }) - this.viewer.cameraHandler.controls.addEventListener('wake', () => { - this.pipeline.onStationaryEnd() - }) } public update(deltaTime: number) { + this.viewer.cameraHandler.controls.update(deltaTime) this.batcher.update(deltaTime) const viewer = new Vector3() @@ -289,50 +288,43 @@ export default class SpeckleRenderer { this.viewer.cameraHandler.activeCam.camera.updateProjectionMatrix() this.viewer.cameraHandler.camera.updateProjectionMatrix() + const currentAzimuth = this.viewer.cameraHandler.controls.azimuthAngle + const currentPolar = this.viewer.cameraHandler.controls.polarAngle + const currentPosition = this.viewer.cameraHandler.activeCam.camera.position + const dAzimuth = Math.max(0, Math.abs(currentAzimuth - this.lastAzimuth) - 0.0001) + const dPolar = Math.max(0, Math.abs(currentPolar - this.lastPolar) - 0.0001) + const dPosition = Math.max( + 0, + currentPosition.distanceTo(this.lastCameraPosition) - 0.001 + ) + const motionMaxDelta = Math.max(0, Math.max(dAzimuth, dPolar, dPosition)) + if (motionMaxDelta === 0 && this.lastCameraMotionDelta > 0) { + this.pipeline.onStationaryBegin() + } + if (motionMaxDelta > 0 && this.lastCameraMotionDelta === 0) { + this._needsRender = true + this.pipeline.onStationaryEnd() + } + this.lastCameraMotionDelta = motionMaxDelta + this.lastAzimuth = currentAzimuth + this.lastPolar = currentPolar + this.lastCameraPosition.copy(currentPosition) + this.pipeline.update(this) - - // const currentAzimuth = this.viewer.cameraHandler.controls.azimuthAngle - // const currentPolar = this.viewer.cameraHandler.controls.polarAngle - // const currentDistance = this.viewer.cameraHandler.controls.distance - // const currentTarget = this.viewer.cameraHandler.controls.getTarget(new Vector3()) - // /** This looks so much better */ - // const dAzimuth = Math.max( - // 0, - // Math.abs(currentAzimuth - this.lastAzimuth) - this.CAMERA_MOTION_EPSILON - // ) - // const dPolar = Math.max( - // 0, - // Math.abs(currentPolar - this.lastPolar) - this.CAMERA_MOTION_EPSILON - // ) - // const dDistance = Math.max( - // 0, - // Math.abs(currentDistance - this.lastDistance) - this.CAMERA_MOTION_EPSILON - // ) - // const dTarget = currentTarget.distanceTo(this.lastTarget) - - // const motionMaxDelta = Math.max(0, Math.max(dAzimuth, dPolar, dDistance, dTarget)) - // if (motionMaxDelta === 0 && this.lasMaxCameraMotion > 0) { - // console.log('Stopped') - // } - // this.lasMaxCameraMotion = motionMaxDelta - - // this.lastAzimuth = currentAzimuth - // this.lastPolar = currentPolar - // this.lastDistance = currentDistance - // this.lastTarget.copy(currentTarget) } - public render(camera: Camera): boolean { - camera - this.batcher.render(this.renderer) - const needsRender = this.pipeline.render() - return needsRender - // this.renderer.render(this.scene, camera) + public render(): void { + if (this._needsRender) { + this.batcher.render(this.renderer) + this._needsRender = this.pipeline.render() + // this.renderer.render(this.scene, camera) + } } public resize(width: number, height: number) { this.renderer.setSize(width, height) this.pipeline.resize(width, height) + this._needsRender = true } public addRenderTree(subtreeId: string) { diff --git a/packages/viewer/src/modules/Viewer.ts b/packages/viewer/src/modules/Viewer.ts index d474201cd..93829b213 100644 --- a/packages/viewer/src/modules/Viewer.ts +++ b/packages/viewer/src/modules/Viewer.ts @@ -148,18 +148,13 @@ export class Viewer extends EventEmitter implements IViewer { private update() { const delta = this.clock.getDelta() - this.needsRender = this.cameraHandler.controls.update(delta) this.speckleRenderer.update(delta) this.stats?.update() requestAnimationFrame(this.frame.bind(this)) } private render() { - if (this.needsRender) { - this._needsRender = this.speckleRenderer.render( - this.cameraHandler.activeCam.camera - ) - } + this.speckleRenderer.render() } public async init(): Promise { diff --git a/packages/viewer/src/modules/pipeline/Pipeline.ts b/packages/viewer/src/modules/pipeline/Pipeline.ts index 02ea0fe40..85bdf3e53 100644 --- a/packages/viewer/src/modules/pipeline/Pipeline.ts +++ b/packages/viewer/src/modules/pipeline/Pipeline.ts @@ -313,6 +313,10 @@ export class Pipeline { } public onStationaryBegin() { + if (this.renderType === RenderType.ACCUMULATION) { + this.accumulationFrame = 0 + return + } this.renderType = RenderType.ACCUMULATION this.accumulationFrame = 0 this.depthPass.enabled = true @@ -326,6 +330,7 @@ export class Pipeline { } public onStationaryEnd() { + if (this.renderType === RenderType.NORMAL) return this.renderType = RenderType.NORMAL this.staticAoPass.enabled = false this.applySaoPass.enabled = true From 6eced47c44eea2162f6456d3d044647c4dacd31f Mon Sep 17 00:00:00 2001 From: AlexandruPopovici Date: Mon, 17 Oct 2022 14:32:00 +0300 Subject: [PATCH 12/54] Implemented interpolating during frame accumulation between the dynamic SAO and the static AO. This makes the transition much more smooth and seamless than simply jumping straight to the static AO which begins accumulating from 0 --- .../shaders/speckle-apply-ao-frag.ts | 19 +++++++ .../shaders/speckle-apply-ao-vert.ts | 6 ++ .../{ApplySAOPass.ts => ApplyAOPass.ts} | 56 ++++++++++++++++--- .../viewer/src/modules/pipeline/Pipeline.ts | 10 +++- .../src/modules/pipeline/SpecklePass.ts | 3 + 5 files changed, 85 insertions(+), 9 deletions(-) create mode 100644 packages/viewer/src/modules/materials/shaders/speckle-apply-ao-frag.ts create mode 100644 packages/viewer/src/modules/materials/shaders/speckle-apply-ao-vert.ts rename packages/viewer/src/modules/pipeline/{ApplySAOPass.ts => ApplyAOPass.ts} (51%) diff --git a/packages/viewer/src/modules/materials/shaders/speckle-apply-ao-frag.ts b/packages/viewer/src/modules/materials/shaders/speckle-apply-ao-frag.ts new file mode 100644 index 000000000..c8e2b2211 --- /dev/null +++ b/packages/viewer/src/modules/materials/shaders/speckle-apply-ao-frag.ts @@ -0,0 +1,19 @@ +export const speckleApplyAoFrag = ` + uniform float opacity; + uniform sampler2D tDiffuse; + uniform sampler2D tDiffuseInterp; + varying vec2 vUv; + #if ACCUMULATE == 1 + uniform float frameIndex; + #endif + + void main() { + vec3 currentSample = texture2D( tDiffuse, vUv ).rgb; + vec3 interpSample = texture2D( tDiffuseInterp, vUv ).rgb; + #if ACCUMULATE == 1 + gl_FragColor.rgb = mix(interpSample, currentSample, frameIndex/float(NUM_FRAMES)); + #else + gl_FragColor.rgb = currentSample; + #endif + gl_FragColor.a = 1.; + }` diff --git a/packages/viewer/src/modules/materials/shaders/speckle-apply-ao-vert.ts b/packages/viewer/src/modules/materials/shaders/speckle-apply-ao-vert.ts new file mode 100644 index 000000000..7d241ade3 --- /dev/null +++ b/packages/viewer/src/modules/materials/shaders/speckle-apply-ao-vert.ts @@ -0,0 +1,6 @@ +export const speckleApplyAoVert = ` + varying vec2 vUv; + void main() { + vUv = uv; + gl_Position = projectionMatrix * modelViewMatrix * vec4( position, 1.0 ); + }` diff --git a/packages/viewer/src/modules/pipeline/ApplySAOPass.ts b/packages/viewer/src/modules/pipeline/ApplyAOPass.ts similarity index 51% rename from packages/viewer/src/modules/pipeline/ApplySAOPass.ts rename to packages/viewer/src/modules/pipeline/ApplyAOPass.ts index a378f3c45..ad1ca0fd9 100644 --- a/packages/viewer/src/modules/pipeline/ApplySAOPass.ts +++ b/packages/viewer/src/modules/pipeline/ApplyAOPass.ts @@ -1,28 +1,43 @@ import { AddEquation, + Camera, CustomBlending, DstAlphaFactor, DstColorFactor, NoBlending, + Scene, ShaderMaterial, Texture, - UniformsUtils, ZeroFactor } from 'three' import { FullScreenQuad, Pass } from 'three/examples/jsm/postprocessing/Pass' -import { CopyShader } from 'three/examples/jsm/shaders/CopyShader.js' -import { InputColorTextureUniform, SpecklePass } from './SpecklePass' +import { speckleApplyAoFrag } from '../materials/shaders/speckle-apply-ao-frag' +import { speckleApplyAoVert } from '../materials/shaders/speckle-apply-ao-vert' +import { Pipeline, RenderType } from './Pipeline' +import { + InputColorTextureUniform, + InputColorInterpolateTextureUniform, + SpeckleProgressivePass +} from './SpecklePass' -export class ApplySAOPass extends Pass implements SpecklePass { +export class ApplySAOPass extends Pass implements SpeckleProgressivePass { private fsQuad: FullScreenQuad public materialCopy: ShaderMaterial + private frameIndex = 0 constructor() { super() this.materialCopy = new ShaderMaterial({ - uniforms: UniformsUtils.clone(CopyShader.uniforms), - vertexShader: CopyShader.vertexShader, - fragmentShader: CopyShader.fragmentShader, + defines: { + ACCUMULATE: 0 + }, + uniforms: { + tDiffuse: { value: null }, + tDiffuseInterp: { value: null }, + frameIndex: { value: 0 } + }, + vertexShader: speckleApplyAoVert, + fragmentShader: speckleApplyAoFrag, blending: NoBlending }) this.materialCopy.transparent = true @@ -40,7 +55,10 @@ export class ApplySAOPass extends Pass implements SpecklePass { this.fsQuad = new FullScreenQuad(this.materialCopy) } - public setTexture(uName: InputColorTextureUniform, texture: Texture) { + public setTexture( + uName: InputColorTextureUniform | InputColorInterpolateTextureUniform, + texture: Texture + ) { this.materialCopy.uniforms[uName].value = texture this.materialCopy.needsUpdate = true } @@ -57,6 +75,28 @@ export class ApplySAOPass extends Pass implements SpecklePass { params } + setFrameIndex(index: number) { + this.frameIndex = index + } + + setRenderType(type: RenderType) { + if (type === RenderType.NORMAL) { + this.materialCopy.defines['ACCUMULATE'] = 0 + } else { + this.materialCopy.defines['ACCUMULATE'] = 1 + this.frameIndex = 0 + } + this.materialCopy.needsUpdate = true + } + + public update(scene: Scene, camera: Camera) { + scene + camera + this.materialCopy.defines['NUM_FRAMES'] = Pipeline.ACCUMULATE_FRAMES + this.materialCopy.uniforms['frameIndex'].value = this.frameIndex + this.materialCopy.needsUpdate = true + } + render(renderer, writeBuffer, readBuffer /*, deltaTime, maskActive*/) { writeBuffer readBuffer diff --git a/packages/viewer/src/modules/pipeline/Pipeline.ts b/packages/viewer/src/modules/pipeline/Pipeline.ts index 85bdf3e53..6a6d69d2d 100644 --- a/packages/viewer/src/modules/pipeline/Pipeline.ts +++ b/packages/viewer/src/modules/pipeline/Pipeline.ts @@ -3,7 +3,7 @@ import { EffectComposer } from 'three/examples/jsm/postprocessing/EffectComposer import { RenderPass } from 'three/examples/jsm/postprocessing/RenderPass.js' import Batcher from '../batching/Batcher' import SpeckleRenderer from '../SpeckleRenderer' -import { ApplySAOPass } from './ApplySAOPass' +import { ApplySAOPass } from './ApplyAOPass' import { CopyOutputPass } from './CopyOutputPass' import { DepthPass } from './DepthPass' import { NormalsPass } from './NormalsPass' @@ -257,6 +257,7 @@ export class Pipeline { this.dynamicAoPass.setTexture('tDepth', this.depthPass.outputTexture) this.dynamicAoPass.setTexture('tNormal', this.normalsPass.outputTexture) this.applySaoPass.setTexture('tDiffuse', this.dynamicAoPass.outputTexture) + this.applySaoPass.setTexture('tDiffuseInterp', this.dynamicAoPass.outputTexture) this.staticAoPass.setTexture('tDepth', this.depthPass.outputTexture) let restoreVisibility @@ -288,7 +289,10 @@ export class Pipeline { this.dynamicAoPass.update(renderer.scene, renderer.camera) this.normalsPass.update(renderer.scene, renderer.camera) this.staticAoPass.update(renderer.scene, renderer.camera) + this.applySaoPass.update(renderer.scene, renderer.camera) + this.staticAoPass.setFrameIndex(this.accumulationFrame) + this.applySaoPass.setFrameIndex(this.accumulationFrame) } public render(): boolean { @@ -326,16 +330,20 @@ export class Pipeline { this.applySaoPass.enabled = true this.staticAoPass.enabled = true this.applySaoPass.setTexture('tDiffuse', this.staticAoPass.outputTexture) + this.applySaoPass.setTexture('tDiffuseInterp', this.dynamicAoPass.outputTexture) + this.applySaoPass.setRenderType(this.renderType) console.warn('Starting stationary') } public onStationaryEnd() { if (this.renderType === RenderType.NORMAL) return + this.accumulationFrame = 0 this.renderType = RenderType.NORMAL this.staticAoPass.enabled = false this.applySaoPass.enabled = true this.dynamicAoPass.enabled = true this.applySaoPass.setTexture('tDiffuse', this.dynamicAoPass.outputTexture) + this.applySaoPass.setRenderType(this.renderType) console.warn('Ending stationary') } } diff --git a/packages/viewer/src/modules/pipeline/SpecklePass.ts b/packages/viewer/src/modules/pipeline/SpecklePass.ts index 7dc2b2c9b..f014fbec0 100644 --- a/packages/viewer/src/modules/pipeline/SpecklePass.ts +++ b/packages/viewer/src/modules/pipeline/SpecklePass.ts @@ -1,8 +1,10 @@ import { Camera, Plane, Scene, Texture } from 'three' +import { RenderType } from './Pipeline' export type InputColorTextureUniform = 'tDiffuse' export type InputDepthTextureUniform = 'tDepth' export type InputNormalsTextureUniform = 'tNormal' +export type InputColorInterpolateTextureUniform = 'tDiffuseInterp' export interface SpecklePass { onBeforeRender?: () => void @@ -19,4 +21,5 @@ export interface SpecklePass { export interface SpeckleProgressivePass extends SpecklePass { setFrameIndex(index: number) + setRenderType?(type: RenderType) } From 3b802412890d6e2a344f1fafce081380ff497203 Mon Sep 17 00:00:00 2001 From: AlexandruPopovici Date: Mon, 17 Oct 2022 15:21:51 +0300 Subject: [PATCH 13/54] Added configurable intensity to static AO. Fixed an issue with static scenario detection where the accumulation frames were ran twice in succession, because the 'camera controller' is probably clamping the angles to X decimal points at the end of a camera motion cycle, which in turn triggers a camera move larger than our defined epsilon --- packages/viewer-sandbox/src/Sandbox.ts | 14 +++++++++++-- packages/viewer-sandbox/src/main.ts | 7 ++----- .../viewer/src/modules/SpeckleRenderer.ts | 20 +++++++++++++++---- .../speckle-static-ao-generate-frag.ts | 5 ++--- .../src/modules/pipeline/StaticAOPass.ts | 5 +++-- 5 files changed, 35 insertions(+), 16 deletions(-) diff --git a/packages/viewer-sandbox/src/Sandbox.ts b/packages/viewer-sandbox/src/Sandbox.ts index 4a26bbc0e..14efb1144 100644 --- a/packages/viewer-sandbox/src/Sandbox.ts +++ b/packages/viewer-sandbox/src/Sandbox.ts @@ -47,8 +47,8 @@ export default class Sandbox { }, staticAoEnabled: true, staticAoParams: { - intensity: 1, - kernelRadius: 0.5, // World space + intensity: 0.8, + kernelRadius: 0.35, // World space kernelSize: 16, minDistance: 0, maxDistance: 0.008 @@ -463,6 +463,16 @@ export default class Sandbox { // this.viewer.getRenderer().pipelineOptions = Sandbox.pipelineParams // this.viewer.requestRender() // }) + staticAoFolder + .addInput(Sandbox.pipelineParams.staticAoParams, 'intensity', { + min: 0, + max: 5, + step: 0.01 + }) + .on('change', () => { + this.viewer.getRenderer().pipelineOptions = Sandbox.pipelineParams + this.viewer.requestRender() + }) staticAoFolder .addInput(Sandbox.pipelineParams.staticAoParams, 'minDistance', { min: 0, diff --git a/packages/viewer-sandbox/src/main.ts b/packages/viewer-sandbox/src/main.ts index 1a1b0dcb8..005d3fddb 100644 --- a/packages/viewer-sandbox/src/main.ts +++ b/packages/viewer-sandbox/src/main.ts @@ -80,14 +80,11 @@ sandbox.makeFilteringUI() // Load demo object await sandbox.loadUrl( - 'https://speckle.xyz/streams/da9e320dad/commits/5388ef24b8?c=%5B-7.66134,10.82932,6.41935,-0.07739,-13.88552,1.8697,0,1%5D' + // 'https://speckle.xyz/streams/da9e320dad/commits/5388ef24b8?c=%5B-7.66134,10.82932,6.41935,-0.07739,-13.88552,1.8697,0,1%5D' // Revit sample house (good for bim-like stuff with many display meshes) - // 'https://speckle.xyz/streams/da9e320dad/commits/5388ef24b8' + 'https://speckle.xyz/streams/da9e320dad/commits/5388ef24b8' // 'Super' heavy revit shit // 'https://speckle.xyz/streams/e6f9156405/commits/0694d53bb5' - // Same sample revit house, local to dim's computer - // 'http://localhost:3000/streams/6960695d7b/commits/da0a2343fa' - // 'http://100.66.180.109:3000/streams/6960695d7b/commits/417526751d' // IFC building (good for a tree based structure) // 'https://latest.speckle.dev/streams/92b620fb17/commits/2ebd336223' // IFC story, a subtree of the above diff --git a/packages/viewer/src/modules/SpeckleRenderer.ts b/packages/viewer/src/modules/SpeckleRenderer.ts index 5611cbc0b..e3e2ee9b0 100644 --- a/packages/viewer/src/modules/SpeckleRenderer.ts +++ b/packages/viewer/src/modules/SpeckleRenderer.ts @@ -48,6 +48,9 @@ import { DefaultPipelineOptions, Pipeline, PipelineOptions } from './pipeline/Pi export default class SpeckleRenderer { private readonly SHOW_HELPERS = false + private readonly ANGLE_EPSILON = 0.0001 + private readonly POSITION_REST_EPSILON = 0.001 + private readonly POSITION_RESUME_EPSILON = 0.01 private _renderer: WebGLRenderer public _scene: Scene private _needsRender: boolean @@ -291,17 +294,26 @@ export default class SpeckleRenderer { const currentAzimuth = this.viewer.cameraHandler.controls.azimuthAngle const currentPolar = this.viewer.cameraHandler.controls.polarAngle const currentPosition = this.viewer.cameraHandler.activeCam.camera.position - const dAzimuth = Math.max(0, Math.abs(currentAzimuth - this.lastAzimuth) - 0.0001) - const dPolar = Math.max(0, Math.abs(currentPolar - this.lastPolar) - 0.0001) + const dAzimuth = Math.max( + 0, + Math.abs(currentAzimuth - this.lastAzimuth) - this.ANGLE_EPSILON + ) + const dPolar = Math.max( + 0, + Math.abs(currentPolar - this.lastPolar) - this.ANGLE_EPSILON + ) const dPosition = Math.max( 0, - currentPosition.distanceTo(this.lastCameraPosition) - 0.001 + currentPosition.distanceTo(this.lastCameraPosition) - this.POSITION_REST_EPSILON ) const motionMaxDelta = Math.max(0, Math.max(dAzimuth, dPolar, dPosition)) if (motionMaxDelta === 0 && this.lastCameraMotionDelta > 0) { this.pipeline.onStationaryBegin() } - if (motionMaxDelta > 0 && this.lastCameraMotionDelta === 0) { + if ( + motionMaxDelta > this.POSITION_RESUME_EPSILON && + this.lastCameraMotionDelta === 0 + ) { this._needsRender = true this.pipeline.onStationaryEnd() } diff --git a/packages/viewer/src/modules/materials/shaders/speckle-static-ao-generate-frag.ts b/packages/viewer/src/modules/materials/shaders/speckle-static-ao-generate-frag.ts index af007d73b..2496cb948 100644 --- a/packages/viewer/src/modules/materials/shaders/speckle-static-ao-generate-frag.ts +++ b/packages/viewer/src/modules/materials/shaders/speckle-static-ao-generate-frag.ts @@ -255,7 +255,7 @@ export const speckleStaticAoGenerateFrag = /* glsl */ ` occlusion += 1.0; } } - return clamp( occlusion / float( KERNEL_SIZE ), 0.0, 1.0 ); + return clamp( occlusion * intensity / float( KERNEL_SIZE ), 0.0, 1.0 ); #endif } void main() { @@ -268,6 +268,5 @@ export const speckleStaticAoGenerateFrag = /* glsl */ ` float ambientOcclusion = getAmbientOcclusion( viewPosition, centerDepth ); gl_FragColor = getDefaultColor( vUv ); gl_FragColor.xyz *= ambientOcclusion; - gl_FragColor.a = 1.;//1. / float(NUM_FRAMES); - // gl_FragColor.xyz = getViewNormal(viewPosition, vUv, centerDepth); + gl_FragColor.a = 1.; }` diff --git a/packages/viewer/src/modules/pipeline/StaticAOPass.ts b/packages/viewer/src/modules/pipeline/StaticAOPass.ts index 96eace04d..90a009688 100644 --- a/packages/viewer/src/modules/pipeline/StaticAOPass.ts +++ b/packages/viewer/src/modules/pipeline/StaticAOPass.ts @@ -48,8 +48,8 @@ export interface StaticAoPassParams { } export const DefaultStaticAoPassParams = { - intensity: 1, - kernelRadius: 0.5, // World space + intensity: 0.8, + kernelRadius: 0.35, // World space kernelSize: 16, minDistance: 0, maxDistance: 0.008 @@ -188,6 +188,7 @@ export class StaticAOPass extends Pass implements SpeckleProgressivePass { this.aoMaterial.uniforms['kernel'].value = this.kernels[this.frameIndex] this.aoMaterial.uniforms['tNoise'].value = this.noiseTextures[this.frameIndex] + this.aoMaterial.uniforms['intensity'].value = this.params.intensity this.aoMaterial.uniforms['kernelRadius'].value = this.params.kernelRadius this.aoMaterial.uniforms['frameIndex'].value = this.frameIndex From 17692baab7e8067b7a60e3928b211b830c449543 Mon Sep 17 00:00:00 2001 From: AlexandruPopovici Date: Mon, 17 Oct 2022 20:06:12 +0300 Subject: [PATCH 14/54] Dynamic AO at half res --- packages/viewer-sandbox/src/main.ts | 4 ++-- packages/viewer/src/modules/SpeckleRenderer.ts | 3 ++- packages/viewer/src/modules/pipeline/DynamicAOPass.ts | 7 +++++-- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/packages/viewer-sandbox/src/main.ts b/packages/viewer-sandbox/src/main.ts index 005d3fddb..01918e9c3 100644 --- a/packages/viewer-sandbox/src/main.ts +++ b/packages/viewer-sandbox/src/main.ts @@ -82,7 +82,7 @@ sandbox.makeFilteringUI() await sandbox.loadUrl( // 'https://speckle.xyz/streams/da9e320dad/commits/5388ef24b8?c=%5B-7.66134,10.82932,6.41935,-0.07739,-13.88552,1.8697,0,1%5D' // Revit sample house (good for bim-like stuff with many display meshes) - 'https://speckle.xyz/streams/da9e320dad/commits/5388ef24b8' + // 'https://speckle.xyz/streams/da9e320dad/commits/5388ef24b8' // 'Super' heavy revit shit // 'https://speckle.xyz/streams/e6f9156405/commits/0694d53bb5' // IFC building (good for a tree based structure) @@ -97,7 +97,7 @@ await sandbox.loadUrl( // AutoCAD // 'https://latest.speckle.dev/streams/3ed8357f29/commits/d10f2af1ce' //Blizzard world - // 'https://latest.speckle.dev/streams/0c6ad366c4/commits/aa1c393aec' + 'https://latest.speckle.dev/streams/0c6ad366c4/commits/aa1c393aec' //Car // 'https://latest.speckle.dev/streams/17d2e25a97/commits/6b6cf3d43e' // Jonathon's diff --git a/packages/viewer/src/modules/SpeckleRenderer.ts b/packages/viewer/src/modules/SpeckleRenderer.ts index e3e2ee9b0..43e9b7cdd 100644 --- a/packages/viewer/src/modules/SpeckleRenderer.ts +++ b/packages/viewer/src/modules/SpeckleRenderer.ts @@ -50,7 +50,7 @@ export default class SpeckleRenderer { private readonly SHOW_HELPERS = false private readonly ANGLE_EPSILON = 0.0001 private readonly POSITION_REST_EPSILON = 0.001 - private readonly POSITION_RESUME_EPSILON = 0.01 + private readonly POSITION_RESUME_EPSILON = 0.001 private _renderer: WebGLRenderer public _scene: Scene private _needsRender: boolean @@ -307,6 +307,7 @@ export default class SpeckleRenderer { currentPosition.distanceTo(this.lastCameraPosition) - this.POSITION_REST_EPSILON ) const motionMaxDelta = Math.max(0, Math.max(dAzimuth, dPolar, dPosition)) + // console.log(motionMaxDelta, this.lastCameraMotionDelta) if (motionMaxDelta === 0 && this.lastCameraMotionDelta > 0) { this.pipeline.onStationaryBegin() } diff --git a/packages/viewer/src/modules/pipeline/DynamicAOPass.ts b/packages/viewer/src/modules/pipeline/DynamicAOPass.ts index a1da743ea..1fe5cba94 100644 --- a/packages/viewer/src/modules/pipeline/DynamicAOPass.ts +++ b/packages/viewer/src/modules/pipeline/DynamicAOPass.ts @@ -54,7 +54,7 @@ export const DefaultDynamicAOPassParams = { bias: 0.15, normalsType: NormalsType.ACCURATE, blurEnabled: true, - blurRadius: 4, + blurRadius: 2, blurStdDev: 4, blurDepthCutoff: 0.0007 } @@ -69,6 +69,7 @@ export class DynamicSAOPass extends Pass implements SpecklePass { private blurIntermediateRenderTarget: WebGLRenderTarget = null private fsQuad: FullScreenQuad = null private _outputType: DynamicAOOutputType = DynamicAOOutputType.AO_BLURRED + private outputScale = 0.5 private prevStdDev: number private prevNumSamples: number @@ -297,7 +298,9 @@ export class DynamicSAOPass extends Pass implements SpecklePass { renderer.setClearAlpha(originalClearAlpha) } - public setSize(width: number, height: number) { + public setSize(inputWidth: number, inputHeight: number) { + const width = inputWidth * this.outputScale + const height = inputHeight * this.outputScale this.saoRenderTarget.setSize(width, height) this.blurIntermediateRenderTarget.setSize(width, height) From bbce671b30c456f110db04beae9429315610a5ff Mon Sep 17 00:00:00 2001 From: AlexandruPopovici Date: Tue, 18 Oct 2022 17:04:10 +0300 Subject: [PATCH 15/54] Some improvements to how we update the rendering pipeline and it's dynamic and progressive states --- packages/viewer-sandbox/src/Sandbox.ts | 2 +- .../viewer/src/modules/pipeline/ColorPass.ts | 15 ++ .../src/modules/pipeline/DynamicAOPass.ts | 2 +- .../viewer/src/modules/pipeline/Pipeline.ts | 209 +++++++++--------- 4 files changed, 125 insertions(+), 103 deletions(-) create mode 100644 packages/viewer/src/modules/pipeline/ColorPass.ts diff --git a/packages/viewer-sandbox/src/Sandbox.ts b/packages/viewer-sandbox/src/Sandbox.ts index 14efb1144..9fdadf4fa 100644 --- a/packages/viewer-sandbox/src/Sandbox.ts +++ b/packages/viewer-sandbox/src/Sandbox.ts @@ -41,7 +41,7 @@ export default class Sandbox { bias: 0.15, normalsType: 2, blurEnabled: true, - blurRadius: 4, + blurRadius: 1, blurStdDev: 4, blurDepthCutoff: 0.0007 }, diff --git a/packages/viewer/src/modules/pipeline/ColorPass.ts b/packages/viewer/src/modules/pipeline/ColorPass.ts new file mode 100644 index 000000000..b83a3c845 --- /dev/null +++ b/packages/viewer/src/modules/pipeline/ColorPass.ts @@ -0,0 +1,15 @@ +import { Camera, Scene, Texture } from 'three' +import { RenderPass } from 'three/examples/jsm/postprocessing/RenderPass' +import { SpecklePass } from './SpecklePass' + +export class ColorPass extends RenderPass implements SpecklePass { + public constructor(scene: Scene, camera: Camera) { + super(scene, camera) + } + public get displayName(): string { + return 'COLOR' + } + public get outputTexture(): Texture { + return null + } +} diff --git a/packages/viewer/src/modules/pipeline/DynamicAOPass.ts b/packages/viewer/src/modules/pipeline/DynamicAOPass.ts index 1fe5cba94..35a48799d 100644 --- a/packages/viewer/src/modules/pipeline/DynamicAOPass.ts +++ b/packages/viewer/src/modules/pipeline/DynamicAOPass.ts @@ -54,7 +54,7 @@ export const DefaultDynamicAOPassParams = { bias: 0.15, normalsType: NormalsType.ACCURATE, blurEnabled: true, - blurRadius: 2, + blurRadius: 1, blurStdDev: 4, blurDepthCutoff: 0.0007 } diff --git a/packages/viewer/src/modules/pipeline/Pipeline.ts b/packages/viewer/src/modules/pipeline/Pipeline.ts index 6a6d69d2d..ede4bd51f 100644 --- a/packages/viewer/src/modules/pipeline/Pipeline.ts +++ b/packages/viewer/src/modules/pipeline/Pipeline.ts @@ -1,6 +1,8 @@ import { Camera, Plane, Scene, Vector2, WebGLRenderer } from 'three' -import { EffectComposer } from 'three/examples/jsm/postprocessing/EffectComposer.js' -import { RenderPass } from 'three/examples/jsm/postprocessing/RenderPass.js' +import { + EffectComposer, + Pass +} from 'three/examples/jsm/postprocessing/EffectComposer.js' import Batcher from '../batching/Batcher' import SpeckleRenderer from '../SpeckleRenderer' import { ApplySAOPass } from './ApplyAOPass' @@ -19,6 +21,8 @@ import { StaticAOPass, StaticAoPassParams } from './StaticAOPass' +import { SpecklePass } from './SpecklePass' +import { ColorPass } from './ColorPass' export enum RenderType { NORMAL, @@ -61,15 +65,15 @@ export class Pipeline { private _renderer: WebGLRenderer = null private _batcher: Batcher = null private _pipelineOptions: PipelineOptions = Object.assign({}, DefaultPipelineOptions) + private _needsProgressive = false private composer: EffectComposer = null private depthPass: DepthPass = null private normalsPass: NormalsPass = null - private renderPass: RenderPass = null + private renderPass: ColorPass = null private dynamicAoPass: DynamicSAOPass = null private applySaoPass: ApplySAOPass = null private copyOutputPass: CopyOutputPass = null - private staticAoPass: StaticAOPass = null private drawingSize: Vector2 = new Vector2() @@ -78,22 +82,6 @@ export class Pipeline { public set pipelineOptions(options: Partial) { Object.assign(this._pipelineOptions, options) - if (options.dynamicAoEnabled) { - this.dynamicAoPass.enabled = true - this.renderPass.enabled = true - this.applySaoPass.enabled = true - this.normalsPass.enabled = - options.dynamicAoParams.normalsType === NormalsType.DEFAULT ? true : false - this.depthPass.enabled = true - this.copyOutputPass.enabled = false - } else { - this.depthPass.enabled = false - this.dynamicAoPass.enabled = false - this.applySaoPass.enabled = false - this.copyOutputPass.enabled = false - this.normalsPass.enabled = false - this.renderPass.enabled = true - } this.dynamicAoPass.setParams(options.dynamicAoParams) this.staticAoPass.setParams(options.staticAoParams) this.accumulationFrame = 0 @@ -103,123 +91,110 @@ export class Pipeline { } public set pipelineOutput(outputType: PipelineOutputType) { + let pipeline = [] + this.clearPipeline() switch (outputType) { case PipelineOutputType.FINAL: - this.dynamicAoPass.enabled = true - this.renderPass.enabled = true - this.applySaoPass.enabled = true - this.normalsPass.enabled = - this._pipelineOptions.dynamicAoParams.normalsType === NormalsType.DEFAULT - ? true - : false - this.depthPass.enabled = true - this.copyOutputPass.enabled = false - this.dynamicAoPass.setOutputType( - this._pipelineOptions.dynamicAoParams.blurEnabled - ? DynamicAOOutputType.AO_BLURRED - : DynamicAOOutputType.AO - ) + pipeline = this.getDefaultPipeline() + this.applySaoPass.setTexture('tDiffuse', this.staticAoPass.outputTexture) + this.applySaoPass.setTexture('tDiffuseInterp', this.dynamicAoPass.outputTexture) + this.needsProgressive = true break case PipelineOutputType.DEPTH_RGBA: - this.dynamicAoPass.enabled = false - this.renderPass.enabled = false - this.applySaoPass.enabled = false - this.normalsPass.enabled = false - this.depthPass.enabled = true - this.copyOutputPass.enabled = true + pipeline.push(this.depthPass) + pipeline.push(this.copyOutputPass) this.copyOutputPass.setTexture('tDiffuse', this.depthPass.outputTexture) this.copyOutputPass.setOutputType(PipelineOutputType.DEPTH_RGBA) + this.needsProgressive = false break case PipelineOutputType.DEPTH: - this.dynamicAoPass.enabled = false - this.renderPass.enabled = false - this.applySaoPass.enabled = false - this.depthPass.enabled = true - this.normalsPass.enabled = false - this.copyOutputPass.enabled = true + pipeline.push(this.depthPass) + pipeline.push(this.copyOutputPass) this.copyOutputPass.setTexture('tDiffuse', this.depthPass.outputTexture) this.copyOutputPass.setOutputType(PipelineOutputType.DEPTH) + this.needsProgressive = false break case PipelineOutputType.COLOR: - this.depthPass.enabled = false - this.dynamicAoPass.enabled = false - this.applySaoPass.enabled = false - this.copyOutputPass.enabled = false - this.normalsPass.enabled = false - this.renderPass.enabled = true + pipeline.push(this.renderPass) break case PipelineOutputType.GEOMETRY_NORMALS: - this.depthPass.enabled = false - this.dynamicAoPass.enabled = false - this.applySaoPass.enabled = false - this.renderPass.enabled = false + pipeline.push(this.normalsPass) + pipeline.push(this.copyOutputPass) this.normalsPass.enabled = true - this.copyOutputPass.enabled = true this.copyOutputPass.setTexture('tDiffuse', this.normalsPass.outputTexture) this.copyOutputPass.setOutputType(PipelineOutputType.GEOMETRY_NORMALS) + this.needsProgressive = false break case PipelineOutputType.RECONSTRUCTED_NORMALS: - this.depthPass.enabled = true + pipeline.push(this.depthPass) + pipeline.push(this.dynamicAoPass) + pipeline.push(this.copyOutputPass) this.dynamicAoPass.enabled = true - this.applySaoPass.enabled = false - this.renderPass.enabled = false - this.normalsPass.enabled = false - this.copyOutputPass.enabled = true + this.dynamicAoPass.setOutputType(DynamicAOOutputType.RECONSTRUCTED_NORMALS) this.copyOutputPass.setTexture('tDiffuse', this.dynamicAoPass.outputTexture) this.copyOutputPass.setOutputType(PipelineOutputType.GEOMETRY_NORMALS) - this.dynamicAoPass.setOutputType(DynamicAOOutputType.RECONSTRUCTED_NORMALS) + this.needsProgressive = false break case PipelineOutputType.DYNAMIC_AO: - this.depthPass.enabled = true - this.dynamicAoPass.enabled = true - this.applySaoPass.enabled = false - this.renderPass.enabled = false + pipeline.push(this.depthPass) + pipeline.push(this.normalsPass) + pipeline.push(this.dynamicAoPass) + pipeline.push(this.copyOutputPass) this.normalsPass.enabled = this._pipelineOptions.dynamicAoParams.normalsType === NormalsType.DEFAULT ? true : false - this.copyOutputPass.enabled = true + this.dynamicAoPass.enabled = true this.copyOutputPass.setTexture('tDiffuse', this.dynamicAoPass.outputTexture) this.copyOutputPass.setOutputType(PipelineOutputType.COLOR) this.dynamicAoPass.setOutputType(DynamicAOOutputType.AO) + this.needsProgressive = false break case PipelineOutputType.DYNAMIC_AO_BLURED: - this.depthPass.enabled = true - this.dynamicAoPass.enabled = true - this.applySaoPass.enabled = false - this.renderPass.enabled = false + pipeline.push(this.depthPass) + pipeline.push(this.normalsPass) + pipeline.push(this.dynamicAoPass) + pipeline.push(this.copyOutputPass) this.normalsPass.enabled = this._pipelineOptions.dynamicAoParams.normalsType === NormalsType.DEFAULT ? true : false - this.copyOutputPass.enabled = true + this.dynamicAoPass.enabled = true this.copyOutputPass.setTexture('tDiffuse', this.dynamicAoPass.outputTexture) this.copyOutputPass.setOutputType(PipelineOutputType.COLOR) this.dynamicAoPass.setOutputType(DynamicAOOutputType.AO_BLURRED) + this.needsProgressive = false break case PipelineOutputType.PROGRESSIVE_AO: - this.depthPass.enabled = true - this.normalsPass.enabled = false - this.dynamicAoPass.enabled = false - this.renderPass.enabled = false - this.applySaoPass.enabled = false - this.staticAoPass.enabled = true - this.copyOutputPass.enabled = true - this.applySaoPass.setTexture('tDiffuse', this.staticAoPass.outputTexture) + pipeline.push(this.depthPass) + // pipeline.push(this.normalsPass) + pipeline.push(this.dynamicAoPass) + pipeline.push(this.staticAoPass) + pipeline.push(this.copyOutputPass) this.copyOutputPass.setTexture('tDiffuse', this.staticAoPass.outputTexture) this.copyOutputPass.setOutputType(PipelineOutputType.COLOR) + this.needsProgressive = true break default: break } + this.setPipeline(pipeline) + } + + public set needsProgressive(value: boolean) { + this._needsProgressive = value + if (!value) this.renderType = RenderType.NORMAL + if (value && this.renderType === RenderType.NORMAL) + this.renderType = RenderType.ACCUMULATION + this.accumulationFrame = 0 } public constructor(renderer: WebGLRenderer, batcher: Batcher) { @@ -233,32 +208,13 @@ export class Pipeline { public configure(scene: Scene, camera: Camera) { this.depthPass = new DepthPass() this.normalsPass = new NormalsPass() - this.normalsPass.enabled = false this.dynamicAoPass = new DynamicSAOPass() - this.renderPass = new RenderPass(scene, camera) - this.renderPass.renderToScreen = true + this.renderPass = new ColorPass(scene, camera) this.applySaoPass = new ApplySAOPass() - this.applySaoPass.renderToScreen = true - this.staticAoPass = new StaticAOPass() - this.staticAoPass.enabled = false this.copyOutputPass = new CopyOutputPass() this.copyOutputPass.renderToScreen = true - this.copyOutputPass.enabled = false - this.composer.addPass(this.depthPass) - this.composer.addPass(this.normalsPass) - this.composer.addPass(this.dynamicAoPass) - this.composer.addPass(this.staticAoPass) - this.composer.addPass(this.renderPass) - this.composer.addPass(this.applySaoPass) - this.composer.addPass(this.copyOutputPass) - - this.dynamicAoPass.setTexture('tDepth', this.depthPass.outputTexture) - this.dynamicAoPass.setTexture('tNormal', this.normalsPass.outputTexture) - this.applySaoPass.setTexture('tDiffuse', this.dynamicAoPass.outputTexture) - this.applySaoPass.setTexture('tDiffuseInterp', this.dynamicAoPass.outputTexture) - this.staticAoPass.setTexture('tDepth', this.depthPass.outputTexture) let restoreVisibility this.depthPass.onBeforeRender = () => { @@ -278,6 +234,51 @@ export class Pipeline { this.normalsPass.onAfterRender = () => { this._batcher.applyVisibility(restoreVisibility) } + + this.setPipeline(this.getDefaultPipeline()) + } + + private getDefaultPipeline(): Array { + this.renderPass.renderToScreen = true + this.normalsPass.enabled = + this._pipelineOptions.dynamicAoParams.normalsType === NormalsType.DEFAULT + ? true + : false + this.dynamicAoPass.setOutputType( + this._pipelineOptions.dynamicAoParams.blurEnabled + ? DynamicAOOutputType.AO_BLURRED + : DynamicAOOutputType.AO + ) + this.applySaoPass.renderToScreen = true + + this.dynamicAoPass.setTexture('tDepth', this.depthPass.outputTexture) + this.dynamicAoPass.setTexture('tNormal', this.normalsPass.outputTexture) + this.applySaoPass.setTexture('tDiffuse', this.dynamicAoPass.outputTexture) + this.applySaoPass.setTexture('tDiffuseInterp', this.dynamicAoPass.outputTexture) + this.staticAoPass.setTexture('tDepth', this.depthPass.outputTexture) + + const pipeline = [] + pipeline.push(this.depthPass) + pipeline.push(this.normalsPass) + pipeline.push(this.dynamicAoPass) + pipeline.push(this.staticAoPass) + pipeline.push(this.renderPass) + pipeline.push(this.applySaoPass) + + this.needsProgressive = true + return pipeline + } + + private clearPipeline() { + while (this.composer.passes.length > 0) { + this.composer.removePass(this.composer.passes[0]) + } + } + + private setPipeline(pipeline: Array) { + for (let k = 0; k < pipeline.length; k++) { + this.composer.addPass(pipeline[k] as unknown as Pass) + } } public updateClippingPlanes(planes: Plane[]) { @@ -317,12 +318,14 @@ export class Pipeline { } public onStationaryBegin() { + if (!this._needsProgressive) return if (this.renderType === RenderType.ACCUMULATION) { this.accumulationFrame = 0 return } this.renderType = RenderType.ACCUMULATION this.accumulationFrame = 0 + this.depthPass.enabled = true this.normalsPass.enabled = false this.dynamicAoPass.enabled = false @@ -331,18 +334,22 @@ export class Pipeline { this.staticAoPass.enabled = true this.applySaoPass.setTexture('tDiffuse', this.staticAoPass.outputTexture) this.applySaoPass.setTexture('tDiffuseInterp', this.dynamicAoPass.outputTexture) + this.applySaoPass.setRenderType(this.renderType) console.warn('Starting stationary') } public onStationaryEnd() { + if (!this._needsProgressive) return if (this.renderType === RenderType.NORMAL) return this.accumulationFrame = 0 this.renderType = RenderType.NORMAL + this.staticAoPass.enabled = false this.applySaoPass.enabled = true this.dynamicAoPass.enabled = true this.applySaoPass.setTexture('tDiffuse', this.dynamicAoPass.outputTexture) + this.applySaoPass.setRenderType(this.renderType) console.warn('Ending stationary') } From 43955c7f1ee5536204c43580d797b5604efdf950 Mon Sep 17 00:00:00 2001 From: AlexandruPopovici Date: Tue, 18 Oct 2022 21:25:36 +0300 Subject: [PATCH 16/54] Trying to fix the issue with artifacts on progressive AO on mac. seems to be depth encoding related. Changed the depth pass to render to 32 float texture to check if the artifacts are gone --- .../src/modules/materials/shaders/speckle-depth-frag.ts | 2 +- .../materials/shaders/speckle-static-ao-generate-frag.ts | 2 +- packages/viewer/src/modules/pipeline/DepthPass.ts | 6 +++++- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/packages/viewer/src/modules/materials/shaders/speckle-depth-frag.ts b/packages/viewer/src/modules/materials/shaders/speckle-depth-frag.ts index 3d5ac0417..f917adced 100644 --- a/packages/viewer/src/modules/materials/shaders/speckle-depth-frag.ts +++ b/packages/viewer/src/modules/materials/shaders/speckle-depth-frag.ts @@ -33,7 +33,7 @@ void main() { #if DEPTH_PACKING == 3200 gl_FragColor = vec4( vec3( 1.0 - fragCoordZ ), opacity ); #elif DEPTH_PACKING == 3201 - gl_FragColor = packDepthToRGBA( fragCoordZ ); + gl_FragColor = vec4(vec3(fragCoordZ), 1.);//packDepthToRGBA( fragCoordZ ); #endif } ` diff --git a/packages/viewer/src/modules/materials/shaders/speckle-static-ao-generate-frag.ts b/packages/viewer/src/modules/materials/shaders/speckle-static-ao-generate-frag.ts index 2496cb948..d790fcfe3 100644 --- a/packages/viewer/src/modules/materials/shaders/speckle-static-ao-generate-frag.ts +++ b/packages/viewer/src/modules/materials/shaders/speckle-static-ao-generate-frag.ts @@ -40,7 +40,7 @@ export const speckleStaticAoGenerateFrag = /* glsl */ ` } float getDepth( const in vec2 screenPosition ) { - return unpackRGBAToDepth( texture2D( tDepth, screenPosition ) ); + return texture2D( tDepth, screenPosition ).x;//unpackRGBAToDepth( texture2D( tDepth, screenPosition ) ); } float getLinearDepth( const in vec2 screenPosition ) { diff --git a/packages/viewer/src/modules/pipeline/DepthPass.ts b/packages/viewer/src/modules/pipeline/DepthPass.ts index 71db4eb07..4974ccc55 100644 --- a/packages/viewer/src/modules/pipeline/DepthPass.ts +++ b/packages/viewer/src/modules/pipeline/DepthPass.ts @@ -2,6 +2,7 @@ import { Camera, Color, DoubleSide, + FloatType, NoBlending, Plane, RGBADepthPacking, @@ -35,7 +36,10 @@ export class DepthPass extends Pass implements SpecklePass { constructor() { super() - this.renderTarget = new WebGLRenderTarget(256, 256) + this.renderTarget = new WebGLRenderTarget(256, 256, { + type: FloatType, + generateMipmaps: false + }) /** On Chromium, on MacOS the 16 bit depth render buffer appears broken. * We're not really using a stencil buffer at all, we're just forcing * three.js to use a 24 bit depth render buffer From 931190f7380edac7d889b38bd83d950a256f53be Mon Sep 17 00:00:00 2001 From: AlexandruPopovici Date: Tue, 18 Oct 2022 21:56:17 +0300 Subject: [PATCH 17/54] Trying out a new float->RGBA encoding and decoding --- .../materials/shaders/speckle-depth-frag.ts | 19 ++++++++++++++++++- .../speckle-static-ao-generate-frag.ts | 13 ++++++++++++- .../viewer/src/modules/pipeline/DepthPass.ts | 6 +----- 3 files changed, 31 insertions(+), 7 deletions(-) diff --git a/packages/viewer/src/modules/materials/shaders/speckle-depth-frag.ts b/packages/viewer/src/modules/materials/shaders/speckle-depth-frag.ts index f917adced..df7ea3d3b 100644 --- a/packages/viewer/src/modules/materials/shaders/speckle-depth-frag.ts +++ b/packages/viewer/src/modules/materials/shaders/speckle-depth-frag.ts @@ -11,6 +11,23 @@ export const speckleDepthFrag = /* glsl */ ` #include #include varying vec2 vHighPrecisionZW; + +highp vec4 encode32(highp float f) { + highp float e =5.0; + + highp float F = abs(f); + highp float Sign = step(0.0,-f); + highp float Exponent = floor(log2(F)); + highp float Mantissa = (exp2(- Exponent) * F); + Exponent = floor(log2(F) + 127.0) + floor(log2(Mantissa)); + highp vec4 rgba; + rgba[0] = 128.0 * Sign + floor(Exponent*exp2(-1.0)); + rgba[1] = 128.0 * mod(Exponent,2.0) + mod(floor(Mantissa*128.0),128.0); + rgba[2] = floor(mod(floor(Mantissa*exp2(23.0 -8.0)),exp2(8.0))); + rgba[3] = floor(exp2(23.0)*mod(Mantissa,exp2(-15.0))); + return rgba / 255.; +} + void main() { #include vec4 diffuseColor = vec4( 1.0 ); @@ -33,7 +50,7 @@ void main() { #if DEPTH_PACKING == 3200 gl_FragColor = vec4( vec3( 1.0 - fragCoordZ ), opacity ); #elif DEPTH_PACKING == 3201 - gl_FragColor = vec4(vec3(fragCoordZ), 1.);//packDepthToRGBA( fragCoordZ ); + gl_FragColor = encode32( fragCoordZ ); #endif } ` diff --git a/packages/viewer/src/modules/materials/shaders/speckle-static-ao-generate-frag.ts b/packages/viewer/src/modules/materials/shaders/speckle-static-ao-generate-frag.ts index d790fcfe3..90e97ea1f 100644 --- a/packages/viewer/src/modules/materials/shaders/speckle-static-ao-generate-frag.ts +++ b/packages/viewer/src/modules/materials/shaders/speckle-static-ao-generate-frag.ts @@ -38,9 +38,18 @@ export const speckleStaticAoGenerateFrag = /* glsl */ ` vec4 getDefaultColor( const in vec2 screenPosition ) { return vec4( 1.0 ); } + + highp float decode32(highp vec4 rgba) { + highp float Sign = 1.0 - step(128.0,rgba[0])*2.0; + highp float Exponent = 2.0 * mod(rgba[0],128.0) + step(128.0,rgba[1]) - 127.0; + highp float Mantissa = mod(rgba[1],128.0)*65536.0 + rgba[2]*256.0 +rgba[3] + float(0x800000); + highp float Result = Sign * exp2(Exponent) * (Mantissa * exp2(-23.0 )); + return Result; + } float getDepth( const in vec2 screenPosition ) { - return texture2D( tDepth, screenPosition ).x;//unpackRGBAToDepth( texture2D( tDepth, screenPosition ) ); + vec4 plm = texture2D( tDepth, screenPosition ) * 255.; + return decode32( plm.rgba ); } float getLinearDepth( const in vec2 screenPosition ) { @@ -183,6 +192,8 @@ export const speckleStaticAoGenerateFrag = /* glsl */ ` #endif } + + float scaleDividedByCameraFar; float minResolutionMultipliedByCameraFar; // moving costly divides into consts diff --git a/packages/viewer/src/modules/pipeline/DepthPass.ts b/packages/viewer/src/modules/pipeline/DepthPass.ts index 4974ccc55..71db4eb07 100644 --- a/packages/viewer/src/modules/pipeline/DepthPass.ts +++ b/packages/viewer/src/modules/pipeline/DepthPass.ts @@ -2,7 +2,6 @@ import { Camera, Color, DoubleSide, - FloatType, NoBlending, Plane, RGBADepthPacking, @@ -36,10 +35,7 @@ export class DepthPass extends Pass implements SpecklePass { constructor() { super() - this.renderTarget = new WebGLRenderTarget(256, 256, { - type: FloatType, - generateMipmaps: false - }) + this.renderTarget = new WebGLRenderTarget(256, 256) /** On Chromium, on MacOS the 16 bit depth render buffer appears broken. * We're not really using a stencil buffer at all, we're just forcing * three.js to use a 24 bit depth render buffer From e778637b94d95fd2b7c0e90995b2ebabd4f477bc Mon Sep 17 00:00:00 2001 From: AlexandruPopovici Date: Tue, 18 Oct 2022 22:12:02 +0300 Subject: [PATCH 18/54] Added ner and far camera planes controls to see how they're values change depth encoding precision --- packages/viewer-sandbox/src/Sandbox.ts | 24 +++++++++ .../viewer/src/modules/SpeckleRenderer.ts | 52 +++++++++---------- .../materials/shaders/speckle-depth-frag.ts | 19 +------ .../speckle-static-ao-generate-frag.ts | 18 +++---- 4 files changed, 59 insertions(+), 54 deletions(-) diff --git a/packages/viewer-sandbox/src/Sandbox.ts b/packages/viewer-sandbox/src/Sandbox.ts index 9fdadf4fa..3f8f78c7b 100644 --- a/packages/viewer-sandbox/src/Sandbox.ts +++ b/packages/viewer-sandbox/src/Sandbox.ts @@ -335,6 +335,30 @@ export default class Sandbox { this.viewer.requestRender() }) + postFolder + .addInput({ near: 0.01 }, 'near', { + min: 0, + max: 2, + step: 0.001 + }) + .on('change', (ev) => { + this.viewer.cameraHandler.activeCam.camera.near = ev.value + this.viewer.cameraHandler.activeCam.camera.updateProjectionMatrix() + this.viewer.requestRender() + }) + + postFolder + .addInput({ far: 10 }, 'far', { + min: 0, + max: 10000, + step: 1 + }) + .on('change', (ev) => { + this.viewer.cameraHandler.activeCam.camera.far = ev.value + this.viewer.cameraHandler.activeCam.camera.updateProjectionMatrix() + this.viewer.requestRender() + }) + postFolder .addInput(Sandbox.sceneParams, 'tonemapping', { options: { diff --git a/packages/viewer/src/modules/SpeckleRenderer.ts b/packages/viewer/src/modules/SpeckleRenderer.ts index 43e9b7cdd..6afa7d1d5 100644 --- a/packages/viewer/src/modules/SpeckleRenderer.ts +++ b/packages/viewer/src/modules/SpeckleRenderer.ts @@ -264,32 +264,32 @@ export default class SpeckleRenderer { } } - const v = new Vector3() - const box = this.sceneBox - const camPos = new Vector3().copy( - this.viewer.cameraHandler.activeCam.camera.position - ) - let d = 0 - v.set(box.min.x, box.min.y, box.min.z) // 000 - d = Math.max(camPos.distanceTo(v), d) - v.set(box.min.x, box.min.y, box.max.z) // 001 - d = Math.max(camPos.distanceTo(v), d) - v.set(box.min.x, box.max.y, box.min.z) // 010 - d = Math.max(camPos.distanceTo(v), d) - v.set(box.min.x, box.max.y, box.max.z) // 011 - d = Math.max(camPos.distanceTo(v), d) - v.set(box.max.x, box.min.y, box.min.z) // 100 - d = Math.max(camPos.distanceTo(v), d) - v.set(box.max.x, box.min.y, box.max.z) // 101 - d = Math.max(camPos.distanceTo(v), d) - v.set(box.max.x, box.max.y, box.min.z) // 110 - d = Math.max(camPos.distanceTo(v), d) - v.set(box.max.x, box.max.y, box.max.z) // 111 - d = Math.max(camPos.distanceTo(v), d) - this.viewer.cameraHandler.camera.far = d - this.viewer.cameraHandler.activeCam.camera.far = d - this.viewer.cameraHandler.activeCam.camera.updateProjectionMatrix() - this.viewer.cameraHandler.camera.updateProjectionMatrix() + // const v = new Vector3() + // const box = this.sceneBox + // const camPos = new Vector3().copy( + // this.viewer.cameraHandler.activeCam.camera.position + // ) + // let d = 0 + // v.set(box.min.x, box.min.y, box.min.z) // 000 + // d = Math.max(camPos.distanceTo(v), d) + // v.set(box.min.x, box.min.y, box.max.z) // 001 + // d = Math.max(camPos.distanceTo(v), d) + // v.set(box.min.x, box.max.y, box.min.z) // 010 + // d = Math.max(camPos.distanceTo(v), d) + // v.set(box.min.x, box.max.y, box.max.z) // 011 + // d = Math.max(camPos.distanceTo(v), d) + // v.set(box.max.x, box.min.y, box.min.z) // 100 + // d = Math.max(camPos.distanceTo(v), d) + // v.set(box.max.x, box.min.y, box.max.z) // 101 + // d = Math.max(camPos.distanceTo(v), d) + // v.set(box.max.x, box.max.y, box.min.z) // 110 + // d = Math.max(camPos.distanceTo(v), d) + // v.set(box.max.x, box.max.y, box.max.z) // 111 + // d = Math.max(camPos.distanceTo(v), d) + // this.viewer.cameraHandler.camera.far = d + // this.viewer.cameraHandler.activeCam.camera.far = d + // this.viewer.cameraHandler.activeCam.camera.updateProjectionMatrix() + // this.viewer.cameraHandler.camera.updateProjectionMatrix() const currentAzimuth = this.viewer.cameraHandler.controls.azimuthAngle const currentPolar = this.viewer.cameraHandler.controls.polarAngle diff --git a/packages/viewer/src/modules/materials/shaders/speckle-depth-frag.ts b/packages/viewer/src/modules/materials/shaders/speckle-depth-frag.ts index df7ea3d3b..3d5ac0417 100644 --- a/packages/viewer/src/modules/materials/shaders/speckle-depth-frag.ts +++ b/packages/viewer/src/modules/materials/shaders/speckle-depth-frag.ts @@ -11,23 +11,6 @@ export const speckleDepthFrag = /* glsl */ ` #include #include varying vec2 vHighPrecisionZW; - -highp vec4 encode32(highp float f) { - highp float e =5.0; - - highp float F = abs(f); - highp float Sign = step(0.0,-f); - highp float Exponent = floor(log2(F)); - highp float Mantissa = (exp2(- Exponent) * F); - Exponent = floor(log2(F) + 127.0) + floor(log2(Mantissa)); - highp vec4 rgba; - rgba[0] = 128.0 * Sign + floor(Exponent*exp2(-1.0)); - rgba[1] = 128.0 * mod(Exponent,2.0) + mod(floor(Mantissa*128.0),128.0); - rgba[2] = floor(mod(floor(Mantissa*exp2(23.0 -8.0)),exp2(8.0))); - rgba[3] = floor(exp2(23.0)*mod(Mantissa,exp2(-15.0))); - return rgba / 255.; -} - void main() { #include vec4 diffuseColor = vec4( 1.0 ); @@ -50,7 +33,7 @@ void main() { #if DEPTH_PACKING == 3200 gl_FragColor = vec4( vec3( 1.0 - fragCoordZ ), opacity ); #elif DEPTH_PACKING == 3201 - gl_FragColor = encode32( fragCoordZ ); + gl_FragColor = packDepthToRGBA( fragCoordZ ); #endif } ` diff --git a/packages/viewer/src/modules/materials/shaders/speckle-static-ao-generate-frag.ts b/packages/viewer/src/modules/materials/shaders/speckle-static-ao-generate-frag.ts index 90e97ea1f..a455b1116 100644 --- a/packages/viewer/src/modules/materials/shaders/speckle-static-ao-generate-frag.ts +++ b/packages/viewer/src/modules/materials/shaders/speckle-static-ao-generate-frag.ts @@ -38,18 +38,9 @@ export const speckleStaticAoGenerateFrag = /* glsl */ ` vec4 getDefaultColor( const in vec2 screenPosition ) { return vec4( 1.0 ); } - - highp float decode32(highp vec4 rgba) { - highp float Sign = 1.0 - step(128.0,rgba[0])*2.0; - highp float Exponent = 2.0 * mod(rgba[0],128.0) + step(128.0,rgba[1]) - 127.0; - highp float Mantissa = mod(rgba[1],128.0)*65536.0 + rgba[2]*256.0 +rgba[3] + float(0x800000); - highp float Result = Sign * exp2(Exponent) * (Mantissa * exp2(-23.0 )); - return Result; - } float getDepth( const in vec2 screenPosition ) { - vec4 plm = texture2D( tDepth, screenPosition ) * 255.; - return decode32( plm.rgba ); + return unpackRGBAToDepth( texture2D( tDepth, screenPosition ) ); } float getLinearDepth( const in vec2 screenPosition ) { @@ -192,6 +183,13 @@ export const speckleStaticAoGenerateFrag = /* glsl */ ` #endif } + highp float decode32(highp vec4 rgba) { + highp float Sign = 1.0 - step(128.0,rgba[0])*2.0; + highp float Exponent = 2.0 * mod(rgba[0],128.0) + step(128.0,rgba[1]) - 127.0; + highp float Mantissa = mod(rgba[1],128.0)*65536.0 + rgba[2]*256.0 +rgba[3] + float(0x800000); + highp float Result = Sign * exp2(Exponent) * (Mantissa * exp2(-23.0 )); + return Result; + } float scaleDividedByCameraFar; From 60b2f260f92e6f4c0a56cba0aba6600773cabfbb Mon Sep 17 00:00:00 2001 From: AlexandruPopovici Date: Thu, 20 Oct 2022 14:56:13 +0300 Subject: [PATCH 19/54] Finally managed to get AO properly working with encoded linear depth --- packages/viewer-sandbox/src/main.ts | 4 +- .../viewer/src/modules/SpeckleRenderer.ts | 52 +++++++------- .../modules/materials/SpeckleDepthMaterial.ts | 18 +++++ .../materials/shaders/speckle-depth-frag.ts | 29 ++++++-- .../materials/shaders/speckle-depth-vert.ts | 8 +++ .../speckle-static-ao-generate-frag.ts | 71 ++++++++++++++----- .../viewer/src/modules/pipeline/DepthPass.ts | 11 ++- .../viewer/src/modules/pipeline/Pipeline.ts | 14 ++-- .../src/modules/pipeline/StaticAOPass.ts | 2 +- 9 files changed, 150 insertions(+), 59 deletions(-) diff --git a/packages/viewer-sandbox/src/main.ts b/packages/viewer-sandbox/src/main.ts index 01918e9c3..005d3fddb 100644 --- a/packages/viewer-sandbox/src/main.ts +++ b/packages/viewer-sandbox/src/main.ts @@ -82,7 +82,7 @@ sandbox.makeFilteringUI() await sandbox.loadUrl( // 'https://speckle.xyz/streams/da9e320dad/commits/5388ef24b8?c=%5B-7.66134,10.82932,6.41935,-0.07739,-13.88552,1.8697,0,1%5D' // Revit sample house (good for bim-like stuff with many display meshes) - // 'https://speckle.xyz/streams/da9e320dad/commits/5388ef24b8' + 'https://speckle.xyz/streams/da9e320dad/commits/5388ef24b8' // 'Super' heavy revit shit // 'https://speckle.xyz/streams/e6f9156405/commits/0694d53bb5' // IFC building (good for a tree based structure) @@ -97,7 +97,7 @@ await sandbox.loadUrl( // AutoCAD // 'https://latest.speckle.dev/streams/3ed8357f29/commits/d10f2af1ce' //Blizzard world - 'https://latest.speckle.dev/streams/0c6ad366c4/commits/aa1c393aec' + // 'https://latest.speckle.dev/streams/0c6ad366c4/commits/aa1c393aec' //Car // 'https://latest.speckle.dev/streams/17d2e25a97/commits/6b6cf3d43e' // Jonathon's diff --git a/packages/viewer/src/modules/SpeckleRenderer.ts b/packages/viewer/src/modules/SpeckleRenderer.ts index 6afa7d1d5..43e9b7cdd 100644 --- a/packages/viewer/src/modules/SpeckleRenderer.ts +++ b/packages/viewer/src/modules/SpeckleRenderer.ts @@ -264,32 +264,32 @@ export default class SpeckleRenderer { } } - // const v = new Vector3() - // const box = this.sceneBox - // const camPos = new Vector3().copy( - // this.viewer.cameraHandler.activeCam.camera.position - // ) - // let d = 0 - // v.set(box.min.x, box.min.y, box.min.z) // 000 - // d = Math.max(camPos.distanceTo(v), d) - // v.set(box.min.x, box.min.y, box.max.z) // 001 - // d = Math.max(camPos.distanceTo(v), d) - // v.set(box.min.x, box.max.y, box.min.z) // 010 - // d = Math.max(camPos.distanceTo(v), d) - // v.set(box.min.x, box.max.y, box.max.z) // 011 - // d = Math.max(camPos.distanceTo(v), d) - // v.set(box.max.x, box.min.y, box.min.z) // 100 - // d = Math.max(camPos.distanceTo(v), d) - // v.set(box.max.x, box.min.y, box.max.z) // 101 - // d = Math.max(camPos.distanceTo(v), d) - // v.set(box.max.x, box.max.y, box.min.z) // 110 - // d = Math.max(camPos.distanceTo(v), d) - // v.set(box.max.x, box.max.y, box.max.z) // 111 - // d = Math.max(camPos.distanceTo(v), d) - // this.viewer.cameraHandler.camera.far = d - // this.viewer.cameraHandler.activeCam.camera.far = d - // this.viewer.cameraHandler.activeCam.camera.updateProjectionMatrix() - // this.viewer.cameraHandler.camera.updateProjectionMatrix() + const v = new Vector3() + const box = this.sceneBox + const camPos = new Vector3().copy( + this.viewer.cameraHandler.activeCam.camera.position + ) + let d = 0 + v.set(box.min.x, box.min.y, box.min.z) // 000 + d = Math.max(camPos.distanceTo(v), d) + v.set(box.min.x, box.min.y, box.max.z) // 001 + d = Math.max(camPos.distanceTo(v), d) + v.set(box.min.x, box.max.y, box.min.z) // 010 + d = Math.max(camPos.distanceTo(v), d) + v.set(box.min.x, box.max.y, box.max.z) // 011 + d = Math.max(camPos.distanceTo(v), d) + v.set(box.max.x, box.min.y, box.min.z) // 100 + d = Math.max(camPos.distanceTo(v), d) + v.set(box.max.x, box.min.y, box.max.z) // 101 + d = Math.max(camPos.distanceTo(v), d) + v.set(box.max.x, box.max.y, box.min.z) // 110 + d = Math.max(camPos.distanceTo(v), d) + v.set(box.max.x, box.max.y, box.max.z) // 111 + d = Math.max(camPos.distanceTo(v), d) + this.viewer.cameraHandler.camera.far = d + this.viewer.cameraHandler.activeCam.camera.far = d + this.viewer.cameraHandler.activeCam.camera.updateProjectionMatrix() + this.viewer.cameraHandler.camera.updateProjectionMatrix() const currentAzimuth = this.viewer.cameraHandler.controls.azimuthAngle const currentPolar = this.viewer.cameraHandler.controls.polarAngle diff --git a/packages/viewer/src/modules/materials/SpeckleDepthMaterial.ts b/packages/viewer/src/modules/materials/SpeckleDepthMaterial.ts index 3a905f119..3643318b9 100644 --- a/packages/viewer/src/modules/materials/SpeckleDepthMaterial.ts +++ b/packages/viewer/src/modules/materials/SpeckleDepthMaterial.ts @@ -25,6 +25,8 @@ class SpeckleDepthMaterial extends MeshDepthMaterial { this.userData.rteModelViewMatrix = { value: new Matrix4() } + this.userData.near = { value: 0 } + this.userData.far = { value: 0 } ;(this as any).vertProgram = speckleDepthVert ;(this as any).fragProgram = speckleDepthFrag ;(this as any).uniforms = UniformsUtils.merge([ @@ -38,6 +40,12 @@ class SpeckleDepthMaterial extends MeshDepthMaterial { }, rteModelViewMatrix: { value: this.userData.rteModelViewMatrix.value + }, + near: { + value: this.userData.near.value + }, + far: { + value: this.userData.far.value } } ]) @@ -46,6 +54,8 @@ class SpeckleDepthMaterial extends MeshDepthMaterial { shader.uniforms.uViewer_high = this.userData.uViewer_high shader.uniforms.uViewer_low = this.userData.uViewer_low shader.uniforms.rteModelViewMatrix = this.userData.rteModelViewMatrix + shader.uniforms.near = this.userData.near + shader.uniforms.far = this.userData.far shader.vertexShader = this.vertProgram shader.fragmentShader = this.fragProgram } @@ -69,6 +79,8 @@ class SpeckleDepthMaterial extends MeshDepthMaterial { ret.userData.uViewer_high = this.userData.uViewer_high ret.userData.uViewer_low = this.userData.uViewer_low ret.userData.rteModelViewMatrix = this.userData.rteModelViewMatrix + ret.userData.near = this.userData.near + ret.userData.far = this.userData.far return ret } @@ -84,6 +96,12 @@ class SpeckleDepthMaterial extends MeshDepthMaterial { this.userData.rteModelViewMatrix = { value: new Matrix4() } + this.userData.near = { + value: 0 + } + this.userData.far = { + value: 0 + } this.defines['USE_RTE'] = ' ' return this diff --git a/packages/viewer/src/modules/materials/shaders/speckle-depth-frag.ts b/packages/viewer/src/modules/materials/shaders/speckle-depth-frag.ts index 3d5ac0417..756a24030 100644 --- a/packages/viewer/src/modules/materials/shaders/speckle-depth-frag.ts +++ b/packages/viewer/src/modules/materials/shaders/speckle-depth-frag.ts @@ -2,6 +2,11 @@ export const speckleDepthFrag = /* glsl */ ` #if DEPTH_PACKING == 3200 uniform float opacity; #endif +#ifdef LINEAR_DEPTH + varying vec4 vViewPosition; + uniform float near; + uniform float far; +#endif #include #include #include @@ -11,6 +16,15 @@ export const speckleDepthFrag = /* glsl */ ` #include #include varying vec2 vHighPrecisionZW; + +vec3 packLinearDepth(const in float depth) { + const vec3 code = vec3(1.0, 255.0, 65025.0); + vec3 pack = vec3(code * depth); + pack.gb = fract(pack.gb); + pack.rg -= pack.gb * (1.0 / 256.0); + return pack; +} + void main() { #include vec4 diffuseColor = vec4( 1.0 ); @@ -29,11 +43,16 @@ void main() { #endif #include // Higher precision equivalent of gl_FragCoord.z. This assumes depthRange has been left to its default values. - float fragCoordZ = (0.5 * vHighPrecisionZW[0] / vHighPrecisionZW[1] + 0.5); - #if DEPTH_PACKING == 3200 - gl_FragColor = vec4( vec3( 1.0 - fragCoordZ ), opacity ); - #elif DEPTH_PACKING == 3201 - gl_FragColor = packDepthToRGBA( fragCoordZ ); + #ifdef LINEAR_DEPTH + /** View z is negative moving away from the camera */ + gl_FragColor = packDepthToRGBA((vViewPosition.z + near) / (near - far)); + #else + float fragCoordZ = (0.5 * vHighPrecisionZW[0] / vHighPrecisionZW[1] + 0.5); + #if DEPTH_PACKING == 3200 + gl_FragColor = vec4( vec3( 1.0 - fragCoordZ ), opacity ); + #elif DEPTH_PACKING == 3201 + gl_FragColor = packDepthToRGBA( fragCoordZ ); + #endif #endif } ` diff --git a/packages/viewer/src/modules/materials/shaders/speckle-depth-vert.ts b/packages/viewer/src/modules/materials/shaders/speckle-depth-vert.ts index 28db0f04c..c15b357e5 100644 --- a/packages/viewer/src/modules/materials/shaders/speckle-depth-vert.ts +++ b/packages/viewer/src/modules/materials/shaders/speckle-depth-vert.ts @@ -7,6 +7,9 @@ export const speckleDepthVert = /* glsl */ ` uniform vec3 uViewer_low; uniform mat4 rteModelViewMatrix; #endif +#ifdef LINEAR_DEPTH + varying vec4 vViewPosition; +#endif #include #include #include @@ -50,8 +53,13 @@ void main() { mvPosition = instanceMatrix * mvPosition; #endif + mvPosition = rteModelViewMatrix * mvPosition; + #ifdef LINEAR_DEPTH + vViewPosition = mvPosition; + #endif + gl_Position = projectionMatrix * mvPosition; #include // #include diff --git a/packages/viewer/src/modules/materials/shaders/speckle-static-ao-generate-frag.ts b/packages/viewer/src/modules/materials/shaders/speckle-static-ao-generate-frag.ts index a455b1116..b7724264d 100644 --- a/packages/viewer/src/modules/materials/shaders/speckle-static-ao-generate-frag.ts +++ b/packages/viewer/src/modules/materials/shaders/speckle-static-ao-generate-frag.ts @@ -29,9 +29,9 @@ export const speckleStaticAoGenerateFrag = /* glsl */ ` // #define NUM_FRAMES 16 - #define NORMAL_TEXTURE 0 + #define NORMAL_TEXTURE 1 #define IMPROVED_NORMAL_RECONSTRUCTION 0 - #define ACCURATE_NORMAL_RECONSTRUCTION 1 + #define ACCURATE_NORMAL_RECONSTRUCTION 0 // RGBA depth #include @@ -39,8 +39,24 @@ export const speckleStaticAoGenerateFrag = /* glsl */ ` return vec4( 1.0 ); } + float decode24(const in vec3 x) { + const vec3 decode = 1.0 / vec3(1.0, 255.0, 65025.0); + return dot(x, decode); + } + float getDepth( const in vec2 screenPosition ) { - return unpackRGBAToDepth( texture2D( tDepth, screenPosition ) ); + return texture2D( tDepth, screenPosition ).y;//unpackRGBAToDepth( texture2D( tDepth, screenPosition ) ); + } + + float getPerspectiveDepth(const in vec2 coords) { + float linearDepth = unpackRGBAToDepth( texture2D( tDepth, coords ) ); + float convertedViewZ = orthographicDepthToViewZ(linearDepth, cameraNear, cameraFar); + float centerDepth = viewZToPerspectiveDepth(convertedViewZ, cameraNear, cameraFar); + return centerDepth; + } + + float getViewDepth(const in float linearDepth) { + return orthographicDepthToViewZ(linearDepth, cameraNear, cameraFar); } float getLinearDepth( const in vec2 screenPosition ) { @@ -114,21 +130,21 @@ export const speckleStaticAoGenerateFrag = /* glsl */ ` highp vec2 ddx = vec2(dd.x, 0.); highp vec2 ddy = vec2(0., dd.y); - float sampleDepth = getDepth( uv - ddy ); + float sampleDepth = getPerspectiveDepth( uv - ddy ); float sampleViewZ = getViewZ( sampleDepth ); highp vec3 top = getViewPosition( uv - ddy, sampleDepth, sampleViewZ ); - sampleDepth = getDepth( uv + ddy ); + sampleDepth = getPerspectiveDepth( uv + ddy ); sampleViewZ = getViewZ( sampleDepth ); highp vec3 bottom = getViewPosition( uv + ddy, sampleDepth, sampleViewZ ); highp vec3 center = origin; - sampleDepth = getDepth( uv - ddx ); + sampleDepth = getPerspectiveDepth( uv - ddx ); sampleViewZ = getViewZ( sampleDepth ); highp vec3 left = getViewPosition( uv - ddx, sampleDepth, sampleViewZ ); - sampleDepth = getDepth( uv + ddx ); + sampleDepth = getPerspectiveDepth( uv + ddx ); sampleViewZ = getViewZ( sampleDepth ); highp vec3 right = getViewPosition( uv + ddx, sampleDepth, sampleViewZ ); @@ -140,18 +156,18 @@ export const speckleStaticAoGenerateFrag = /* glsl */ ` // get depth values at 1 & 2 pixels offsets from current along the horizontal axis vec4 H = vec4( - getDepth(uv - ddx), - getDepth(uv + ddx), - getDepth(uv - 2. * ddx), - getDepth(uv + 2. * ddx) + getPerspectiveDepth(uv - ddx), + getPerspectiveDepth(uv + ddx), + getPerspectiveDepth(uv - 2. * ddx), + getPerspectiveDepth(uv + 2. * ddx) ); // get depth values at 1 & 2 pixels offsets from current along the vertical axis vec4 V = vec4( - getDepth(uv - ddy), - getDepth(uv + ddy), - getDepth(uv - 2. * ddy), - getDepth(uv + 2. * ddy) + getPerspectiveDepth(uv - ddy), + getPerspectiveDepth(uv + ddy), + getPerspectiveDepth(uv - 2. * ddy), + getPerspectiveDepth(uv + 2. * ddy) ); // current pixel's depth difference from slope of offset depth samples @@ -257,7 +273,8 @@ export const speckleStaticAoGenerateFrag = /* glsl */ ` vec4 samplePointNDC = cameraProjectionMatrix * vec4( samplePoint, 1.0 ); // project point and calculate NDC samplePointNDC /= samplePointNDC.w; vec2 samplePointUv = samplePointNDC.xy * 0.5 + 0.5; // compute uv coordinates - float realDepth = getLinearDepth( samplePointUv ); // get linear depth from depth texture + float realDepth = unpackRGBAToDepth( texture2D( tDepth, samplePointUv ) );//getLinearDepth( samplePointUv ); // get linear depth from depth texture + // float realDepth = getLinearDepth( samplePointUv ); // get linear depth from depth texture float sampleDepth = viewZToOrthographicDepth( samplePoint.z, cameraNear, cameraFar ); // compute linear depth of the sample view Z value float delta = sampleDepth - realDepth; if ( delta > minDistance && delta < maxDistance ) { // if fragment is before sample point, increase occlusion @@ -268,14 +285,32 @@ export const speckleStaticAoGenerateFrag = /* glsl */ ` #endif } void main() { - float centerDepth = getDepth( vUv ); + float linearDepth = unpackRGBAToDepth( texture2D( tDepth, vUv ) ); + // float convertedViewZ = orthographicDepthToViewZ(linearDepth, cameraNear, cameraFar); + // float centerDepth = viewZToPerspectiveDepth(convertedViewZ, cameraNear, cameraFar); + float centerDepth = getPerspectiveDepth(vUv); if( centerDepth >= ( 1.0 - EPSILON ) ) { discard; } - float centerViewZ = getViewZ( centerDepth ); + float centerViewZ = getViewDepth(linearDepth); vec3 viewPosition = getViewPosition( vUv, centerDepth, centerViewZ ); + vec3 viewNormal = getViewNormal(viewPosition, vUv, centerDepth); float ambientOcclusion = getAmbientOcclusion( viewPosition, centerDepth ); gl_FragColor = getDefaultColor( vUv ); gl_FragColor.xyz *= ambientOcclusion; gl_FragColor.a = 1.; + // gl_FragColor.xyz = vec3(linearDepth);//vec3(getDepth( vUv ));//vec3((viewPosition.z + cameraNear) / (cameraNear-cameraFar)); + + // float centerDepth = getDepth(vUv);//getDepth( vUv ); + // if( centerDepth >= ( 1.0 - EPSILON ) ) { + // discard; + // } + // float centerViewZ = getViewZ( centerDepth ); + // vec3 viewPosition = getViewPosition( vUv, centerDepth, centerViewZ ); + // vec3 viewNormal = getViewNormal(viewPosition, vUv, centerDepth); + // float ambientOcclusion = getAmbientOcclusion( viewPosition, centerDepth ); + // gl_FragColor = getDefaultColor( vUv ); + // gl_FragColor.xyz *= ambientOcclusion; + // gl_FragColor.a = 1.; + // // gl_FragColor.xyz = vec3(centerDepth);//vec3(getDepth( vUv ));//vec3((viewPosition.z + cameraNear) / (cameraNear-cameraFar)); }` diff --git a/packages/viewer/src/modules/pipeline/DepthPass.ts b/packages/viewer/src/modules/pipeline/DepthPass.ts index 71db4eb07..63ed6e5e4 100644 --- a/packages/viewer/src/modules/pipeline/DepthPass.ts +++ b/packages/viewer/src/modules/pipeline/DepthPass.ts @@ -3,6 +3,8 @@ import { Color, DoubleSide, NoBlending, + OrthographicCamera, + PerspectiveCamera, Plane, RGBADepthPacking, Scene, @@ -47,7 +49,7 @@ export class DepthPass extends Pass implements SpecklePass { { depthPacking: RGBADepthPacking }, - ['USE_RTE', 'ALPHATEST_REJECTION'] + ['USE_RTE', 'ALPHATEST_REJECTION', 'LINEAR_DEPTH'] ) this.depthMaterial.blending = NoBlending this.depthMaterial.side = DoubleSide @@ -60,6 +62,13 @@ export class DepthPass extends Pass implements SpecklePass { public update(scene: Scene, camera: Camera) { this.camera = camera this.scene = scene + this.depthMaterial.userData.near.value = ( + camera as PerspectiveCamera | OrthographicCamera + ).near + this.depthMaterial.userData.far.value = ( + camera as PerspectiveCamera | OrthographicCamera + ).far + this.depthMaterial.needsUpdate = true } public render(renderer, writeBuffer, readBuffer) { diff --git a/packages/viewer/src/modules/pipeline/Pipeline.ts b/packages/viewer/src/modules/pipeline/Pipeline.ts index ede4bd51f..ed637f986 100644 --- a/packages/viewer/src/modules/pipeline/Pipeline.ts +++ b/packages/viewer/src/modules/pipeline/Pipeline.ts @@ -175,10 +175,11 @@ export class Pipeline { case PipelineOutputType.PROGRESSIVE_AO: pipeline.push(this.depthPass) - // pipeline.push(this.normalsPass) + pipeline.push(this.normalsPass) pipeline.push(this.dynamicAoPass) pipeline.push(this.staticAoPass) pipeline.push(this.copyOutputPass) + this.normalsPass.enabled this.copyOutputPass.setTexture('tDiffuse', this.staticAoPass.outputTexture) this.copyOutputPass.setOutputType(PipelineOutputType.COLOR) this.needsProgressive = true @@ -240,10 +241,10 @@ export class Pipeline { private getDefaultPipeline(): Array { this.renderPass.renderToScreen = true - this.normalsPass.enabled = - this._pipelineOptions.dynamicAoParams.normalsType === NormalsType.DEFAULT - ? true - : false + this.normalsPass.enabled = true + // this._pipelineOptions.dynamicAoParams.normalsType === NormalsType.DEFAULT + // ? true + // : false this.dynamicAoPass.setOutputType( this._pipelineOptions.dynamicAoParams.blurEnabled ? DynamicAOOutputType.AO_BLURRED @@ -256,6 +257,7 @@ export class Pipeline { this.applySaoPass.setTexture('tDiffuse', this.dynamicAoPass.outputTexture) this.applySaoPass.setTexture('tDiffuseInterp', this.dynamicAoPass.outputTexture) this.staticAoPass.setTexture('tDepth', this.depthPass.outputTexture) + this.staticAoPass.setTexture('tNormal', this.normalsPass.outputTexture) const pipeline = [] pipeline.push(this.depthPass) @@ -327,7 +329,7 @@ export class Pipeline { this.accumulationFrame = 0 this.depthPass.enabled = true - this.normalsPass.enabled = false + // this.normalsPass.enabled = false this.dynamicAoPass.enabled = false this.renderPass.enabled = true this.applySaoPass.enabled = true diff --git a/packages/viewer/src/modules/pipeline/StaticAOPass.ts b/packages/viewer/src/modules/pipeline/StaticAOPass.ts index 90a009688..5248ef42c 100644 --- a/packages/viewer/src/modules/pipeline/StaticAOPass.ts +++ b/packages/viewer/src/modules/pipeline/StaticAOPass.ts @@ -117,7 +117,7 @@ export class StaticAOPass extends Pass implements SpeckleProgressivePass { tNoise: { value: null }, kernel: { value: null }, - minDistance: { value: 0.0 }, + minDistance: { value: 0.001 }, maxDistance: { value: 1 } } }) From f33faf8475b0eb2fa87c8c0cab27c5bdfd1f434b Mon Sep 17 00:00:00 2001 From: AlexandruPopovici Date: Thu, 20 Oct 2022 15:07:13 +0300 Subject: [PATCH 20/54] Trying 24 bit encoding --- .../modules/materials/shaders/speckle-depth-frag.ts | 10 +++++++++- .../shaders/speckle-static-ao-generate-frag.ts | 13 +++++++------ 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/packages/viewer/src/modules/materials/shaders/speckle-depth-frag.ts b/packages/viewer/src/modules/materials/shaders/speckle-depth-frag.ts index 756a24030..bd230e37e 100644 --- a/packages/viewer/src/modules/materials/shaders/speckle-depth-frag.ts +++ b/packages/viewer/src/modules/materials/shaders/speckle-depth-frag.ts @@ -25,6 +25,14 @@ vec3 packLinearDepth(const in float depth) { return pack; } +vec3 encode24(const in float x) { + const vec3 code = vec3(1.0, 255.0, 65025.0); + vec3 pack = vec3(code * x); + pack.gb = fract(pack.gb); + pack.rg -= pack.gb * (1.0 / 256.0); + return pack; +} + void main() { #include vec4 diffuseColor = vec4( 1.0 ); @@ -45,7 +53,7 @@ void main() { // Higher precision equivalent of gl_FragCoord.z. This assumes depthRange has been left to its default values. #ifdef LINEAR_DEPTH /** View z is negative moving away from the camera */ - gl_FragColor = packDepthToRGBA((vViewPosition.z + near) / (near - far)); + gl_FragColor = vec4(encode24((vViewPosition.z + near) / (near - far)), 1.); #else float fragCoordZ = (0.5 * vHighPrecisionZW[0] / vHighPrecisionZW[1] + 0.5); #if DEPTH_PACKING == 3200 diff --git a/packages/viewer/src/modules/materials/shaders/speckle-static-ao-generate-frag.ts b/packages/viewer/src/modules/materials/shaders/speckle-static-ao-generate-frag.ts index b7724264d..8c5f4b678 100644 --- a/packages/viewer/src/modules/materials/shaders/speckle-static-ao-generate-frag.ts +++ b/packages/viewer/src/modules/materials/shaders/speckle-static-ao-generate-frag.ts @@ -39,10 +39,6 @@ export const speckleStaticAoGenerateFrag = /* glsl */ ` return vec4( 1.0 ); } - float decode24(const in vec3 x) { - const vec3 decode = 1.0 / vec3(1.0, 255.0, 65025.0); - return dot(x, decode); - } float getDepth( const in vec2 screenPosition ) { return texture2D( tDepth, screenPosition ).y;//unpackRGBAToDepth( texture2D( tDepth, screenPosition ) ); @@ -207,6 +203,11 @@ export const speckleStaticAoGenerateFrag = /* glsl */ ` return Result; } + float decode24(const in vec3 x) { + const vec3 decode = 1.0 / vec3(1.0, 255.0, 65025.0); + return dot(x, decode); + } + float scaleDividedByCameraFar; float minResolutionMultipliedByCameraFar; @@ -273,7 +274,7 @@ export const speckleStaticAoGenerateFrag = /* glsl */ ` vec4 samplePointNDC = cameraProjectionMatrix * vec4( samplePoint, 1.0 ); // project point and calculate NDC samplePointNDC /= samplePointNDC.w; vec2 samplePointUv = samplePointNDC.xy * 0.5 + 0.5; // compute uv coordinates - float realDepth = unpackRGBAToDepth( texture2D( tDepth, samplePointUv ) );//getLinearDepth( samplePointUv ); // get linear depth from depth texture + float realDepth = decode24( texture2D( tDepth, samplePointUv ).rgb );//getLinearDepth( samplePointUv ); // get linear depth from depth texture // float realDepth = getLinearDepth( samplePointUv ); // get linear depth from depth texture float sampleDepth = viewZToOrthographicDepth( samplePoint.z, cameraNear, cameraFar ); // compute linear depth of the sample view Z value float delta = sampleDepth - realDepth; @@ -285,7 +286,7 @@ export const speckleStaticAoGenerateFrag = /* glsl */ ` #endif } void main() { - float linearDepth = unpackRGBAToDepth( texture2D( tDepth, vUv ) ); + float linearDepth = decode24( texture2D( tDepth, vUv ).rgb ); // float convertedViewZ = orthographicDepthToViewZ(linearDepth, cameraNear, cameraFar); // float centerDepth = viewZToPerspectiveDepth(convertedViewZ, cameraNear, cameraFar); float centerDepth = getPerspectiveDepth(vUv); From 72f3850c9f893a220b92aa6c9f651f86380a777e Mon Sep 17 00:00:00 2001 From: Alex Date: Thu, 20 Oct 2022 22:23:12 +0300 Subject: [PATCH 21/54] Fix for the macOS artifacts using linear depth buffer and a bias value --- packages/viewer-sandbox/src/Sandbox.ts | 12 +++ packages/viewer-sandbox/src/main.ts | 4 +- .../materials/shaders/speckle-depth-frag.ts | 18 +--- .../speckle-static-ao-generate-frag.ts | 86 +++++-------------- .../viewer/src/modules/pipeline/Pipeline.ts | 12 ++- .../src/modules/pipeline/StaticAOPass.ts | 3 + 6 files changed, 46 insertions(+), 89 deletions(-) diff --git a/packages/viewer-sandbox/src/Sandbox.ts b/packages/viewer-sandbox/src/Sandbox.ts index 3f8f78c7b..de0a526ce 100644 --- a/packages/viewer-sandbox/src/Sandbox.ts +++ b/packages/viewer-sandbox/src/Sandbox.ts @@ -50,6 +50,7 @@ export default class Sandbox { intensity: 0.8, kernelRadius: 0.35, // World space kernelSize: 16, + bias: 0.01, minDistance: 0, maxDistance: 0.008 } @@ -528,6 +529,17 @@ export default class Sandbox { this.viewer.requestRender() }) + staticAoFolder + .addInput(Sandbox.pipelineParams.staticAoParams, 'bias', { + min: -1, + max: 1, + step: 0.0001 + }) + .on('change', () => { + this.viewer.getRenderer().pipelineOptions = Sandbox.pipelineParams + this.viewer.requestRender() + }) + staticAoFolder .addInput(Sandbox.pipelineParams.staticAoParams, 'kernelSize', { min: 1, diff --git a/packages/viewer-sandbox/src/main.ts b/packages/viewer-sandbox/src/main.ts index 005d3fddb..00a7cdb84 100644 --- a/packages/viewer-sandbox/src/main.ts +++ b/packages/viewer-sandbox/src/main.ts @@ -82,11 +82,11 @@ sandbox.makeFilteringUI() await sandbox.loadUrl( // 'https://speckle.xyz/streams/da9e320dad/commits/5388ef24b8?c=%5B-7.66134,10.82932,6.41935,-0.07739,-13.88552,1.8697,0,1%5D' // Revit sample house (good for bim-like stuff with many display meshes) - 'https://speckle.xyz/streams/da9e320dad/commits/5388ef24b8' + // 'https://speckle.xyz/streams/da9e320dad/commits/5388ef24b8' // 'Super' heavy revit shit // 'https://speckle.xyz/streams/e6f9156405/commits/0694d53bb5' // IFC building (good for a tree based structure) - // 'https://latest.speckle.dev/streams/92b620fb17/commits/2ebd336223' + 'https://latest.speckle.dev/streams/92b620fb17/commits/2ebd336223' // IFC story, a subtree of the above // 'https://latest.speckle.dev/streams/92b620fb17/objects/8247bbc53865b0e0cb5ee4e252e66216' // Small scale lines diff --git a/packages/viewer/src/modules/materials/shaders/speckle-depth-frag.ts b/packages/viewer/src/modules/materials/shaders/speckle-depth-frag.ts index bd230e37e..f3b144f99 100644 --- a/packages/viewer/src/modules/materials/shaders/speckle-depth-frag.ts +++ b/packages/viewer/src/modules/materials/shaders/speckle-depth-frag.ts @@ -17,22 +17,6 @@ export const speckleDepthFrag = /* glsl */ ` #include varying vec2 vHighPrecisionZW; -vec3 packLinearDepth(const in float depth) { - const vec3 code = vec3(1.0, 255.0, 65025.0); - vec3 pack = vec3(code * depth); - pack.gb = fract(pack.gb); - pack.rg -= pack.gb * (1.0 / 256.0); - return pack; -} - -vec3 encode24(const in float x) { - const vec3 code = vec3(1.0, 255.0, 65025.0); - vec3 pack = vec3(code * x); - pack.gb = fract(pack.gb); - pack.rg -= pack.gb * (1.0 / 256.0); - return pack; -} - void main() { #include vec4 diffuseColor = vec4( 1.0 ); @@ -53,7 +37,7 @@ void main() { // Higher precision equivalent of gl_FragCoord.z. This assumes depthRange has been left to its default values. #ifdef LINEAR_DEPTH /** View z is negative moving away from the camera */ - gl_FragColor = vec4(encode24((vViewPosition.z + near) / (near - far)), 1.); + gl_FragColor = packDepthToRGBA((vViewPosition.z + near) / (near - far)); #else float fragCoordZ = (0.5 * vHighPrecisionZW[0] / vHighPrecisionZW[1] + 0.5); #if DEPTH_PACKING == 3200 diff --git a/packages/viewer/src/modules/materials/shaders/speckle-static-ao-generate-frag.ts b/packages/viewer/src/modules/materials/shaders/speckle-static-ao-generate-frag.ts index 8c5f4b678..af9e0ffbc 100644 --- a/packages/viewer/src/modules/materials/shaders/speckle-static-ao-generate-frag.ts +++ b/packages/viewer/src/modules/materials/shaders/speckle-static-ao-generate-frag.ts @@ -29,9 +29,9 @@ export const speckleStaticAoGenerateFrag = /* glsl */ ` // #define NUM_FRAMES 16 - #define NORMAL_TEXTURE 1 + #define NORMAL_TEXTURE 0 #define IMPROVED_NORMAL_RECONSTRUCTION 0 - #define ACCURATE_NORMAL_RECONSTRUCTION 0 + #define ACCURATE_NORMAL_RECONSTRUCTION 1 // RGBA depth #include @@ -40,14 +40,14 @@ export const speckleStaticAoGenerateFrag = /* glsl */ ` } - float getDepth( const in vec2 screenPosition ) { - return texture2D( tDepth, screenPosition ).y;//unpackRGBAToDepth( texture2D( tDepth, screenPosition ) ); + float getLinearDepth( const in vec2 screenPosition ) { + return unpackRGBAToDepth( texture2D( tDepth, screenPosition ) ); } float getPerspectiveDepth(const in vec2 coords) { float linearDepth = unpackRGBAToDepth( texture2D( tDepth, coords ) ); - float convertedViewZ = orthographicDepthToViewZ(linearDepth, cameraNear, cameraFar); - float centerDepth = viewZToPerspectiveDepth(convertedViewZ, cameraNear, cameraFar); + float viewZ = orthographicDepthToViewZ(linearDepth, cameraNear, cameraFar); + float centerDepth = viewZToPerspectiveDepth(viewZ, cameraNear, cameraFar); return centerDepth; } @@ -55,16 +55,6 @@ export const speckleStaticAoGenerateFrag = /* glsl */ ` return orthographicDepthToViewZ(linearDepth, cameraNear, cameraFar); } - float getLinearDepth( const in vec2 screenPosition ) { - #if PERSPECTIVE_CAMERA == 1 - float fragCoordZ = getDepth(screenPosition); - float viewZ = perspectiveDepthToViewZ( fragCoordZ, cameraNear, cameraFar ); - return viewZToOrthographicDepth( viewZ, cameraNear, cameraFar ); - #else - return texture2D( tDepth, screenPosition ).x; - #endif - } - float getViewZ( const in float depth ) { #if PERSPECTIVE_CAMERA == 1 return perspectiveDepthToViewZ( depth, cameraNear, cameraFar ); @@ -87,21 +77,21 @@ export const speckleStaticAoGenerateFrag = /* glsl */ ` highp vec2 ddx = vec2(dd.x, 0.); highp vec2 ddy = vec2(0., dd.y); - float sampleDepth = getDepth( uv - ddy ); + float sampleDepth = getPerspectiveDepth( uv - ddy ); float sampleViewZ = getViewZ( sampleDepth ); highp vec3 top = getViewPosition( uv - ddy, sampleDepth, sampleViewZ ); - sampleDepth = getDepth( uv + ddy ); + sampleDepth = getPerspectiveDepth( uv + ddy ); sampleViewZ = getViewZ( sampleDepth ); highp vec3 bottom = getViewPosition( uv + ddy, sampleDepth, sampleViewZ ); highp vec3 center = origin; - sampleDepth = getDepth( uv - ddx ); + sampleDepth = getPerspectiveDepth( uv - ddx ); sampleViewZ = getViewZ( sampleDepth ); highp vec3 left = getViewPosition( uv - ddx, sampleDepth, sampleViewZ ); - sampleDepth = getDepth( uv + ddx ); + sampleDepth = getPerspectiveDepth( uv + ddx ); sampleViewZ = getViewZ( sampleDepth ); highp vec3 right = getViewPosition( uv + ddx, sampleDepth, sampleViewZ ); @@ -152,18 +142,18 @@ export const speckleStaticAoGenerateFrag = /* glsl */ ` // get depth values at 1 & 2 pixels offsets from current along the horizontal axis vec4 H = vec4( - getPerspectiveDepth(uv - ddx), - getPerspectiveDepth(uv + ddx), - getPerspectiveDepth(uv - 2. * ddx), - getPerspectiveDepth(uv + 2. * ddx) + getLinearDepth(uv - ddx), + getLinearDepth(uv + ddx), + getLinearDepth(uv - 2. * ddx), + getLinearDepth(uv + 2. * ddx) ); // get depth values at 1 & 2 pixels offsets from current along the vertical axis vec4 V = vec4( - getPerspectiveDepth(uv - ddy), - getPerspectiveDepth(uv + ddy), - getPerspectiveDepth(uv - 2. * ddy), - getPerspectiveDepth(uv + 2. * ddy) + getLinearDepth(uv - ddy), + getLinearDepth(uv + ddy), + getLinearDepth(uv - 2. * ddy), + getLinearDepth(uv + 2. * ddy) ); // current pixel's depth difference from slope of offset depth samples @@ -195,19 +185,6 @@ export const speckleStaticAoGenerateFrag = /* glsl */ ` #endif } - highp float decode32(highp vec4 rgba) { - highp float Sign = 1.0 - step(128.0,rgba[0])*2.0; - highp float Exponent = 2.0 * mod(rgba[0],128.0) + step(128.0,rgba[1]) - 127.0; - highp float Mantissa = mod(rgba[1],128.0)*65536.0 + rgba[2]*256.0 +rgba[3] + float(0x800000); - highp float Result = Sign * exp2(Exponent) * (Mantissa * exp2(-23.0 )); - return Result; - } - - float decode24(const in vec3 x) { - const vec3 decode = 1.0 / vec3(1.0, 255.0, 65025.0); - return dot(x, decode); - } - float scaleDividedByCameraFar; float minResolutionMultipliedByCameraFar; @@ -230,7 +207,7 @@ export const speckleStaticAoGenerateFrag = /* glsl */ ` vec2 radius = (kernelRadius / size) * pow( alpha, 1.1 ); vec2 sampleUv = vUv + vec2( cos( angle + frameIndex * offset ), sin( angle + frameIndex * offset ) ) * radius; - float sampleDepth = getDepth( sampleUv ); + float sampleDepth = getPerspectiveDepth( sampleUv ); if( sampleDepth >= ( 1.0 - EPSILON ) ) { continue; } @@ -274,9 +251,8 @@ export const speckleStaticAoGenerateFrag = /* glsl */ ` vec4 samplePointNDC = cameraProjectionMatrix * vec4( samplePoint, 1.0 ); // project point and calculate NDC samplePointNDC /= samplePointNDC.w; vec2 samplePointUv = samplePointNDC.xy * 0.5 + 0.5; // compute uv coordinates - float realDepth = decode24( texture2D( tDepth, samplePointUv ).rgb );//getLinearDepth( samplePointUv ); // get linear depth from depth texture - // float realDepth = getLinearDepth( samplePointUv ); // get linear depth from depth texture - float sampleDepth = viewZToOrthographicDepth( samplePoint.z, cameraNear, cameraFar ); // compute linear depth of the sample view Z value + float realDepth = unpackRGBAToDepth( texture2D( tDepth, samplePointUv ) );//getLinearDepth( samplePointUv ); // get linear depth from depth texture + float sampleDepth = viewZToOrthographicDepth( samplePoint.z + bias, cameraNear, cameraFar ); // compute linear depth of the sample view Z value float delta = sampleDepth - realDepth; if ( delta > minDistance && delta < maxDistance ) { // if fragment is before sample point, increase occlusion occlusion += 1.0; @@ -286,32 +262,16 @@ export const speckleStaticAoGenerateFrag = /* glsl */ ` #endif } void main() { - float linearDepth = decode24( texture2D( tDepth, vUv ).rgb ); - // float convertedViewZ = orthographicDepthToViewZ(linearDepth, cameraNear, cameraFar); - // float centerDepth = viewZToPerspectiveDepth(convertedViewZ, cameraNear, cameraFar); + float linearDepth = unpackRGBAToDepth( texture2D( tDepth, vUv ) ); float centerDepth = getPerspectiveDepth(vUv); if( centerDepth >= ( 1.0 - EPSILON ) ) { discard; } float centerViewZ = getViewDepth(linearDepth); vec3 viewPosition = getViewPosition( vUv, centerDepth, centerViewZ ); - vec3 viewNormal = getViewNormal(viewPosition, vUv, centerDepth); + vec3 viewNormal = getViewNormal(viewPosition, vUv, linearDepth); float ambientOcclusion = getAmbientOcclusion( viewPosition, centerDepth ); gl_FragColor = getDefaultColor( vUv ); gl_FragColor.xyz *= ambientOcclusion; gl_FragColor.a = 1.; - // gl_FragColor.xyz = vec3(linearDepth);//vec3(getDepth( vUv ));//vec3((viewPosition.z + cameraNear) / (cameraNear-cameraFar)); - - // float centerDepth = getDepth(vUv);//getDepth( vUv ); - // if( centerDepth >= ( 1.0 - EPSILON ) ) { - // discard; - // } - // float centerViewZ = getViewZ( centerDepth ); - // vec3 viewPosition = getViewPosition( vUv, centerDepth, centerViewZ ); - // vec3 viewNormal = getViewNormal(viewPosition, vUv, centerDepth); - // float ambientOcclusion = getAmbientOcclusion( viewPosition, centerDepth ); - // gl_FragColor = getDefaultColor( vUv ); - // gl_FragColor.xyz *= ambientOcclusion; - // gl_FragColor.a = 1.; - // // gl_FragColor.xyz = vec3(centerDepth);//vec3(getDepth( vUv ));//vec3((viewPosition.z + cameraNear) / (cameraNear-cameraFar)); }` diff --git a/packages/viewer/src/modules/pipeline/Pipeline.ts b/packages/viewer/src/modules/pipeline/Pipeline.ts index ed637f986..1e4100f9f 100644 --- a/packages/viewer/src/modules/pipeline/Pipeline.ts +++ b/packages/viewer/src/modules/pipeline/Pipeline.ts @@ -175,11 +175,10 @@ export class Pipeline { case PipelineOutputType.PROGRESSIVE_AO: pipeline.push(this.depthPass) - pipeline.push(this.normalsPass) + // pipeline.push(this.normalsPass) pipeline.push(this.dynamicAoPass) pipeline.push(this.staticAoPass) pipeline.push(this.copyOutputPass) - this.normalsPass.enabled this.copyOutputPass.setTexture('tDiffuse', this.staticAoPass.outputTexture) this.copyOutputPass.setOutputType(PipelineOutputType.COLOR) this.needsProgressive = true @@ -241,10 +240,9 @@ export class Pipeline { private getDefaultPipeline(): Array { this.renderPass.renderToScreen = true - this.normalsPass.enabled = true - // this._pipelineOptions.dynamicAoParams.normalsType === NormalsType.DEFAULT - // ? true - // : false + this.normalsPass.enabled = this._pipelineOptions.dynamicAoParams.normalsType === NormalsType.DEFAULT + ? true + : false this.dynamicAoPass.setOutputType( this._pipelineOptions.dynamicAoParams.blurEnabled ? DynamicAOOutputType.AO_BLURRED @@ -329,7 +327,7 @@ export class Pipeline { this.accumulationFrame = 0 this.depthPass.enabled = true - // this.normalsPass.enabled = false + this.normalsPass.enabled = false this.dynamicAoPass.enabled = false this.renderPass.enabled = true this.applySaoPass.enabled = true diff --git a/packages/viewer/src/modules/pipeline/StaticAOPass.ts b/packages/viewer/src/modules/pipeline/StaticAOPass.ts index 5248ef42c..71cd070f4 100644 --- a/packages/viewer/src/modules/pipeline/StaticAOPass.ts +++ b/packages/viewer/src/modules/pipeline/StaticAOPass.ts @@ -43,6 +43,7 @@ export interface StaticAoPassParams { intensity: number kernelRadius: number kernelSize: number + bias: number minDistance: number maxDistance: number } @@ -51,6 +52,7 @@ export const DefaultStaticAoPassParams = { intensity: 0.8, kernelRadius: 0.35, // World space kernelSize: 16, + bias: 0.01, minDistance: 0, maxDistance: 0.008 } @@ -190,6 +192,7 @@ export class StaticAOPass extends Pass implements SpeckleProgressivePass { this.aoMaterial.uniforms['intensity'].value = this.params.intensity this.aoMaterial.uniforms['kernelRadius'].value = this.params.kernelRadius + this.aoMaterial.uniforms['bias'].value = this.params.bias this.aoMaterial.uniforms['frameIndex'].value = this.frameIndex this.aoMaterial.uniforms['minDistance'].value = this.params.minDistance From 01bb086047b2f2c849d3743b234b386c675e9960 Mon Sep 17 00:00:00 2001 From: AlexandruPopovici Date: Tue, 25 Oct 2022 17:37:12 +0300 Subject: [PATCH 22/54] Extended the camera controller class and added/changed a few things so that the controller doesn't mess up our camera movement and also we now rely on the events sent by it to determine stationary vs dynamic camera scenarios. This way we spare the rendered from having to determine camera movement deltas by itslef(only to have it ruined by the camera controller anyway) --- .../viewer/src/modules/SpeckleRenderer.ts | 80 +++-- .../src/modules/context/CameraHanlder.js | 71 ++++- .../materials/shaders/speckle-depth-frag.ts | 8 - .../speckle-static-ao-generate-frag.ts | 81 ++--- .../modules/objects/SpeckleCameraControls.ts | 282 ++++++++++++++++++ .../viewer/src/modules/pipeline/Pipeline.ts | 6 +- 6 files changed, 421 insertions(+), 107 deletions(-) create mode 100644 packages/viewer/src/modules/objects/SpeckleCameraControls.ts diff --git a/packages/viewer/src/modules/SpeckleRenderer.ts b/packages/viewer/src/modules/SpeckleRenderer.ts index 43e9b7cdd..867899ca8 100644 --- a/packages/viewer/src/modules/SpeckleRenderer.ts +++ b/packages/viewer/src/modules/SpeckleRenderer.ts @@ -189,10 +189,32 @@ export default class SpeckleRenderer { camHelper.name = 'CamHelper' helpers.add(camHelper) } + + this.viewer.cameraHandler.controls.restThreshold = 0.001 + this.viewer.cameraHandler.controls.addEventListener('rest', () => { + this._needsRender = true + this.pipeline.onStationaryBegin() + }) + this.viewer.cameraHandler.controls.addEventListener('controlstart', () => { + this._needsRender = true + this.pipeline.onStationaryEnd() + }) + + this.viewer.cameraHandler.controls.addEventListener('controlend', () => { + this._needsRender = true + if (this.viewer.cameraHandler.controls.hasRested) + this.pipeline.onStationaryBegin() + }) + + this.viewer.cameraHandler.controls.addEventListener('control', () => { + this._needsRender = true + this.pipeline.onStationaryEnd() + }) } public update(deltaTime: number) { - this.viewer.cameraHandler.controls.update(deltaTime) + this.needsRender = this.viewer.cameraHandler.controls.update(deltaTime) + this.batcher.update(deltaTime) const viewer = new Vector3() @@ -291,37 +313,31 @@ export default class SpeckleRenderer { this.viewer.cameraHandler.activeCam.camera.updateProjectionMatrix() this.viewer.cameraHandler.camera.updateProjectionMatrix() - const currentAzimuth = this.viewer.cameraHandler.controls.azimuthAngle - const currentPolar = this.viewer.cameraHandler.controls.polarAngle - const currentPosition = this.viewer.cameraHandler.activeCam.camera.position - const dAzimuth = Math.max( - 0, - Math.abs(currentAzimuth - this.lastAzimuth) - this.ANGLE_EPSILON - ) - const dPolar = Math.max( - 0, - Math.abs(currentPolar - this.lastPolar) - this.ANGLE_EPSILON - ) - const dPosition = Math.max( - 0, - currentPosition.distanceTo(this.lastCameraPosition) - this.POSITION_REST_EPSILON - ) - const motionMaxDelta = Math.max(0, Math.max(dAzimuth, dPolar, dPosition)) - // console.log(motionMaxDelta, this.lastCameraMotionDelta) - if (motionMaxDelta === 0 && this.lastCameraMotionDelta > 0) { - this.pipeline.onStationaryBegin() - } - if ( - motionMaxDelta > this.POSITION_RESUME_EPSILON && - this.lastCameraMotionDelta === 0 - ) { - this._needsRender = true - this.pipeline.onStationaryEnd() - } - this.lastCameraMotionDelta = motionMaxDelta - this.lastAzimuth = currentAzimuth - this.lastPolar = currentPolar - this.lastCameraPosition.copy(currentPosition) + // const currentAzimuth = this.viewer.cameraHandler.controls.azimuthAngle + // const currentPolar = this.viewer.cameraHandler.controls.polarAngle + // const currentPosition = this.viewer.cameraHandler.activeCam.camera.position + // const dAzimuth = Math.max(0, Math.abs(currentAzimuth - this.lastAzimuth)) + // const dPolar = Math.max(0, Math.abs(currentPolar - this.lastPolar)) + // const dPosition = Math.max(0, currentPosition.distanceTo(this.lastCameraPosition)) + // const motionMaxDelta = Math.max(0, Math.max(dAzimuth, dPolar, dPosition)) + // // console.log(dPosition, this.lastCameraMotionDelta) + // console.log( + // currentPosition.distanceTo(this.viewer.cameraHandler.controls['_targetEnd']) + // ) + // if (motionMaxDelta === 0 && this.lastCameraMotionDelta > 0) { + // this.pipeline.onStationaryBegin() + // } + // if ( + // motionMaxDelta > this.POSITION_RESUME_EPSILON && + // this.lastCameraMotionDelta === 0 + // ) { + // this._needsRender = true + // this.pipeline.onStationaryEnd() + // } + // this.lastCameraMotionDelta = motionMaxDelta + // this.lastAzimuth = currentAzimuth + // this.lastPolar = currentPolar + // this.lastCameraPosition.copy(currentPosition) this.pipeline.update(this) } diff --git a/packages/viewer/src/modules/context/CameraHanlder.js b/packages/viewer/src/modules/context/CameraHanlder.js index 40729c57f..e1b7b935b 100644 --- a/packages/viewer/src/modules/context/CameraHanlder.js +++ b/packages/viewer/src/modules/context/CameraHanlder.js @@ -1,6 +1,7 @@ import * as THREE from 'three' import CameraControls from 'camera-controls' -import { KeyboardKeyHold } from 'hold-event' +import { HOLD_EVENT_TYPE, KeyboardKeyHold } from 'hold-event' +import { SpeckleCameraControls } from '../objects/SpeckleCameraControls' export default class CameraHandler { constructor(viewer) { @@ -30,7 +31,8 @@ export default class CameraHandler { this.orthoCamera.updateProjectionMatrix() CameraControls.install({ THREE }) - this.controls = new CameraControls(this.camera, this.viewer.container) + SpeckleCameraControls.install() + this.controls = new SpeckleCameraControls(this.camera, this.viewer.container) this.controls.maxPolarAngle = Math.PI / 2 this.setupWASDControls() @@ -185,38 +187,103 @@ export default class CameraHandler { const aKey = new KeyboardKeyHold(KEYCODE.A, 16.666) const sKey = new KeyboardKeyHold(KEYCODE.S, 16.666) const dKey = new KeyboardKeyHold(KEYCODE.D, 16.666) + const isTruckingGroup = new Array(4) + + const setTrucking = (index, value) => { + isTruckingGroup[index] = value + if (isTruckingGroup.every((element) => element === false)) { + this.controls.isTrucking = false + this.controls.dispatchEvent({ type: 'rest' }) + } else this.controls.isTrucking = true + } + + aKey.addEventListener( + HOLD_EVENT_TYPE.HOLD_START, + function () { + this.controls.dispatchEvent({ type: 'controlstart' }) + }.bind(this) + ) aKey.addEventListener( 'holding', function (event) { if (this.viewer.mouseOverRenderer === false) return + setTrucking(0, true) this.controls.truck(-0.01 * event.deltaTime, 0, false) return }.bind(this) ) + aKey.addEventListener( + HOLD_EVENT_TYPE.HOLD_END, + function () { + setTrucking(0, false) + }.bind(this) + ) + + dKey.addEventListener( + HOLD_EVENT_TYPE.HOLD_START, + function () { + this.controls.dispatchEvent({ type: 'controlstart' }) + }.bind(this) + ) dKey.addEventListener( 'holding', function (event) { if (this.viewer.mouseOverRenderer === false) return + setTrucking(1, true) this.controls.truck(0.01 * event.deltaTime, 0, false) return }.bind(this) ) + dKey.addEventListener( + HOLD_EVENT_TYPE.HOLD_END, + function () { + setTrucking(1, false) + }.bind(this) + ) + + wKey.addEventListener( + HOLD_EVENT_TYPE.HOLD_START, + function () { + this.controls.dispatchEvent({ type: 'controlstart' }) + }.bind(this) + ) wKey.addEventListener( 'holding', function (event) { if (this.viewer.mouseOverRenderer === false) return + setTrucking(2, true) this.controls.forward(0.01 * event.deltaTime, false) return }.bind(this) ) + wKey.addEventListener( + HOLD_EVENT_TYPE.HOLD_END, + function () { + setTrucking(2, false) + }.bind(this) + ) + + sKey.addEventListener( + HOLD_EVENT_TYPE.HOLD_START, + function () { + this.controls.dispatchEvent({ type: 'controlstart' }) + }.bind(this) + ) sKey.addEventListener( 'holding', function (event) { if (this.viewer.mouseOverRenderer === false) return + setTrucking(3, true) this.controls.forward(-0.01 * event.deltaTime, false) return }.bind(this) ) + sKey.addEventListener( + HOLD_EVENT_TYPE.HOLD_END, + function () { + setTrucking(3, false) + }.bind(this) + ) } onWindowResize() { diff --git a/packages/viewer/src/modules/materials/shaders/speckle-depth-frag.ts b/packages/viewer/src/modules/materials/shaders/speckle-depth-frag.ts index 756a24030..f3b144f99 100644 --- a/packages/viewer/src/modules/materials/shaders/speckle-depth-frag.ts +++ b/packages/viewer/src/modules/materials/shaders/speckle-depth-frag.ts @@ -17,14 +17,6 @@ export const speckleDepthFrag = /* glsl */ ` #include varying vec2 vHighPrecisionZW; -vec3 packLinearDepth(const in float depth) { - const vec3 code = vec3(1.0, 255.0, 65025.0); - vec3 pack = vec3(code * depth); - pack.gb = fract(pack.gb); - pack.rg -= pack.gb * (1.0 / 256.0); - return pack; -} - void main() { #include vec4 diffuseColor = vec4( 1.0 ); diff --git a/packages/viewer/src/modules/materials/shaders/speckle-static-ao-generate-frag.ts b/packages/viewer/src/modules/materials/shaders/speckle-static-ao-generate-frag.ts index b7724264d..af9e0ffbc 100644 --- a/packages/viewer/src/modules/materials/shaders/speckle-static-ao-generate-frag.ts +++ b/packages/viewer/src/modules/materials/shaders/speckle-static-ao-generate-frag.ts @@ -29,9 +29,9 @@ export const speckleStaticAoGenerateFrag = /* glsl */ ` // #define NUM_FRAMES 16 - #define NORMAL_TEXTURE 1 + #define NORMAL_TEXTURE 0 #define IMPROVED_NORMAL_RECONSTRUCTION 0 - #define ACCURATE_NORMAL_RECONSTRUCTION 0 + #define ACCURATE_NORMAL_RECONSTRUCTION 1 // RGBA depth #include @@ -39,19 +39,15 @@ export const speckleStaticAoGenerateFrag = /* glsl */ ` return vec4( 1.0 ); } - float decode24(const in vec3 x) { - const vec3 decode = 1.0 / vec3(1.0, 255.0, 65025.0); - return dot(x, decode); - } - float getDepth( const in vec2 screenPosition ) { - return texture2D( tDepth, screenPosition ).y;//unpackRGBAToDepth( texture2D( tDepth, screenPosition ) ); + float getLinearDepth( const in vec2 screenPosition ) { + return unpackRGBAToDepth( texture2D( tDepth, screenPosition ) ); } float getPerspectiveDepth(const in vec2 coords) { float linearDepth = unpackRGBAToDepth( texture2D( tDepth, coords ) ); - float convertedViewZ = orthographicDepthToViewZ(linearDepth, cameraNear, cameraFar); - float centerDepth = viewZToPerspectiveDepth(convertedViewZ, cameraNear, cameraFar); + float viewZ = orthographicDepthToViewZ(linearDepth, cameraNear, cameraFar); + float centerDepth = viewZToPerspectiveDepth(viewZ, cameraNear, cameraFar); return centerDepth; } @@ -59,16 +55,6 @@ export const speckleStaticAoGenerateFrag = /* glsl */ ` return orthographicDepthToViewZ(linearDepth, cameraNear, cameraFar); } - float getLinearDepth( const in vec2 screenPosition ) { - #if PERSPECTIVE_CAMERA == 1 - float fragCoordZ = getDepth(screenPosition); - float viewZ = perspectiveDepthToViewZ( fragCoordZ, cameraNear, cameraFar ); - return viewZToOrthographicDepth( viewZ, cameraNear, cameraFar ); - #else - return texture2D( tDepth, screenPosition ).x; - #endif - } - float getViewZ( const in float depth ) { #if PERSPECTIVE_CAMERA == 1 return perspectiveDepthToViewZ( depth, cameraNear, cameraFar ); @@ -91,21 +77,21 @@ export const speckleStaticAoGenerateFrag = /* glsl */ ` highp vec2 ddx = vec2(dd.x, 0.); highp vec2 ddy = vec2(0., dd.y); - float sampleDepth = getDepth( uv - ddy ); + float sampleDepth = getPerspectiveDepth( uv - ddy ); float sampleViewZ = getViewZ( sampleDepth ); highp vec3 top = getViewPosition( uv - ddy, sampleDepth, sampleViewZ ); - sampleDepth = getDepth( uv + ddy ); + sampleDepth = getPerspectiveDepth( uv + ddy ); sampleViewZ = getViewZ( sampleDepth ); highp vec3 bottom = getViewPosition( uv + ddy, sampleDepth, sampleViewZ ); highp vec3 center = origin; - sampleDepth = getDepth( uv - ddx ); + sampleDepth = getPerspectiveDepth( uv - ddx ); sampleViewZ = getViewZ( sampleDepth ); highp vec3 left = getViewPosition( uv - ddx, sampleDepth, sampleViewZ ); - sampleDepth = getDepth( uv + ddx ); + sampleDepth = getPerspectiveDepth( uv + ddx ); sampleViewZ = getViewZ( sampleDepth ); highp vec3 right = getViewPosition( uv + ddx, sampleDepth, sampleViewZ ); @@ -156,18 +142,18 @@ export const speckleStaticAoGenerateFrag = /* glsl */ ` // get depth values at 1 & 2 pixels offsets from current along the horizontal axis vec4 H = vec4( - getPerspectiveDepth(uv - ddx), - getPerspectiveDepth(uv + ddx), - getPerspectiveDepth(uv - 2. * ddx), - getPerspectiveDepth(uv + 2. * ddx) + getLinearDepth(uv - ddx), + getLinearDepth(uv + ddx), + getLinearDepth(uv - 2. * ddx), + getLinearDepth(uv + 2. * ddx) ); // get depth values at 1 & 2 pixels offsets from current along the vertical axis vec4 V = vec4( - getPerspectiveDepth(uv - ddy), - getPerspectiveDepth(uv + ddy), - getPerspectiveDepth(uv - 2. * ddy), - getPerspectiveDepth(uv + 2. * ddy) + getLinearDepth(uv - ddy), + getLinearDepth(uv + ddy), + getLinearDepth(uv - 2. * ddy), + getLinearDepth(uv + 2. * ddy) ); // current pixel's depth difference from slope of offset depth samples @@ -199,14 +185,6 @@ export const speckleStaticAoGenerateFrag = /* glsl */ ` #endif } - highp float decode32(highp vec4 rgba) { - highp float Sign = 1.0 - step(128.0,rgba[0])*2.0; - highp float Exponent = 2.0 * mod(rgba[0],128.0) + step(128.0,rgba[1]) - 127.0; - highp float Mantissa = mod(rgba[1],128.0)*65536.0 + rgba[2]*256.0 +rgba[3] + float(0x800000); - highp float Result = Sign * exp2(Exponent) * (Mantissa * exp2(-23.0 )); - return Result; - } - float scaleDividedByCameraFar; float minResolutionMultipliedByCameraFar; @@ -229,7 +207,7 @@ export const speckleStaticAoGenerateFrag = /* glsl */ ` vec2 radius = (kernelRadius / size) * pow( alpha, 1.1 ); vec2 sampleUv = vUv + vec2( cos( angle + frameIndex * offset ), sin( angle + frameIndex * offset ) ) * radius; - float sampleDepth = getDepth( sampleUv ); + float sampleDepth = getPerspectiveDepth( sampleUv ); if( sampleDepth >= ( 1.0 - EPSILON ) ) { continue; } @@ -274,8 +252,7 @@ export const speckleStaticAoGenerateFrag = /* glsl */ ` samplePointNDC /= samplePointNDC.w; vec2 samplePointUv = samplePointNDC.xy * 0.5 + 0.5; // compute uv coordinates float realDepth = unpackRGBAToDepth( texture2D( tDepth, samplePointUv ) );//getLinearDepth( samplePointUv ); // get linear depth from depth texture - // float realDepth = getLinearDepth( samplePointUv ); // get linear depth from depth texture - float sampleDepth = viewZToOrthographicDepth( samplePoint.z, cameraNear, cameraFar ); // compute linear depth of the sample view Z value + float sampleDepth = viewZToOrthographicDepth( samplePoint.z + bias, cameraNear, cameraFar ); // compute linear depth of the sample view Z value float delta = sampleDepth - realDepth; if ( delta > minDistance && delta < maxDistance ) { // if fragment is before sample point, increase occlusion occlusion += 1.0; @@ -286,31 +263,15 @@ export const speckleStaticAoGenerateFrag = /* glsl */ ` } void main() { float linearDepth = unpackRGBAToDepth( texture2D( tDepth, vUv ) ); - // float convertedViewZ = orthographicDepthToViewZ(linearDepth, cameraNear, cameraFar); - // float centerDepth = viewZToPerspectiveDepth(convertedViewZ, cameraNear, cameraFar); float centerDepth = getPerspectiveDepth(vUv); if( centerDepth >= ( 1.0 - EPSILON ) ) { discard; } float centerViewZ = getViewDepth(linearDepth); vec3 viewPosition = getViewPosition( vUv, centerDepth, centerViewZ ); - vec3 viewNormal = getViewNormal(viewPosition, vUv, centerDepth); + vec3 viewNormal = getViewNormal(viewPosition, vUv, linearDepth); float ambientOcclusion = getAmbientOcclusion( viewPosition, centerDepth ); gl_FragColor = getDefaultColor( vUv ); gl_FragColor.xyz *= ambientOcclusion; gl_FragColor.a = 1.; - // gl_FragColor.xyz = vec3(linearDepth);//vec3(getDepth( vUv ));//vec3((viewPosition.z + cameraNear) / (cameraNear-cameraFar)); - - // float centerDepth = getDepth(vUv);//getDepth( vUv ); - // if( centerDepth >= ( 1.0 - EPSILON ) ) { - // discard; - // } - // float centerViewZ = getViewZ( centerDepth ); - // vec3 viewPosition = getViewPosition( vUv, centerDepth, centerViewZ ); - // vec3 viewNormal = getViewNormal(viewPosition, vUv, centerDepth); - // float ambientOcclusion = getAmbientOcclusion( viewPosition, centerDepth ); - // gl_FragColor = getDefaultColor( vUv ); - // gl_FragColor.xyz *= ambientOcclusion; - // gl_FragColor.a = 1.; - // // gl_FragColor.xyz = vec3(centerDepth);//vec3(getDepth( vUv ));//vec3((viewPosition.z + cameraNear) / (cameraNear-cameraFar)); }` diff --git a/packages/viewer/src/modules/objects/SpeckleCameraControls.ts b/packages/viewer/src/modules/objects/SpeckleCameraControls.ts new file mode 100644 index 000000000..c7467322b --- /dev/null +++ b/packages/viewer/src/modules/objects/SpeckleCameraControls.ts @@ -0,0 +1,282 @@ +import CameraControls from 'camera-controls' +import { MathUtils, PerspectiveCamera, Vector3 } from 'three' + +let ACTION +;(function (ACTION) { + ACTION[(ACTION['NONE'] = 0)] = 'NONE' + ACTION[(ACTION['ROTATE'] = 1)] = 'ROTATE' + ACTION[(ACTION['TRUCK'] = 2)] = 'TRUCK' + ACTION[(ACTION['OFFSET'] = 3)] = 'OFFSET' + ACTION[(ACTION['DOLLY'] = 4)] = 'DOLLY' + ACTION[(ACTION['ZOOM'] = 5)] = 'ZOOM' + ACTION[(ACTION['TOUCH_ROTATE'] = 6)] = 'TOUCH_ROTATE' + ACTION[(ACTION['TOUCH_TRUCK'] = 7)] = 'TOUCH_TRUCK' + ACTION[(ACTION['TOUCH_OFFSET'] = 8)] = 'TOUCH_OFFSET' + ACTION[(ACTION['TOUCH_DOLLY'] = 9)] = 'TOUCH_DOLLY' + ACTION[(ACTION['TOUCH_ZOOM'] = 10)] = 'TOUCH_ZOOM' + ACTION[(ACTION['TOUCH_DOLLY_TRUCK'] = 11)] = 'TOUCH_DOLLY_TRUCK' + ACTION[(ACTION['TOUCH_DOLLY_OFFSET'] = 12)] = 'TOUCH_DOLLY_OFFSET' + ACTION[(ACTION['TOUCH_ZOOM_TRUCK'] = 13)] = 'TOUCH_ZOOM_TRUCK' + ACTION[(ACTION['TOUCH_ZOOM_OFFSET'] = 14)] = 'TOUCH_ZOOM_OFFSET' +})(ACTION || (ACTION = {})) +function isPerspectiveCamera(camera) { + return camera.isPerspectiveCamera +} +function isOrthographicCamera(camera) { + return camera.isOrthographicCamera +} + +const EPSILON = 1e-5 +function approxZero(number, error = EPSILON) { + return Math.abs(number) < error +} +function approxEquals(a, b, error = EPSILON) { + return approxZero(a - b, error) +} + +let _deltaTarget, _deltaOffset, _v3A, _v3B, _v3C +let _xColumn +let _yColumn +let _zColumn + +export class SpeckleCameraControls extends CameraControls { + private _didDolly = false + public _isTrucking = false + private _hasRestedLastFrame = false + static install() { + _v3A = new Vector3() + _v3B = new Vector3() + _v3C = new Vector3() + _xColumn = new Vector3() + _yColumn = new Vector3() + _zColumn = new Vector3() + _deltaTarget = new Vector3() + _deltaOffset = new Vector3() + } + + public get hasRested() { + return this._hasRested + } + + public set isTrucking(value: boolean) { + this._isTrucking = value + } + + /** + * Dolly in/out camera position to given distance. + * @param distance Distance of dolly. + * @param enableTransition Whether to move smoothly or immediately. + * @category Methods + */ + dollyTo(distance: number, enableTransition = false): Promise { + const lastRadius = this._sphericalEnd.radius + const newRadius = MathUtils.clamp(distance, this.minDistance, this.maxDistance) + const hasCollider = this.colliderMeshes.length >= 1 + + if (hasCollider) { + const maxDistanceByCollisionTest = this._collisionTest() + const isCollided = approxEquals( + maxDistanceByCollisionTest, + this._spherical.radius + ) + const isDollyIn = lastRadius > newRadius + + if (!isDollyIn && isCollided) return Promise.resolve() + + this._sphericalEnd.radius = Math.min(newRadius, maxDistanceByCollisionTest) + } else { + this._sphericalEnd.radius = newRadius + } + + this._needsUpdate = true + + if (!enableTransition) { + this._spherical.radius = this._sphericalEnd.radius + this._didDolly = true + this.dispatchEvent({ type: 'controlstart' }) + } + + const resolveImmediately = + !enableTransition || + approxEquals( + this._spherical.radius, + this._sphericalEnd.radius, + this.restThreshold + ) + return this._createOnRestPromise(resolveImmediately) + } + + update(delta) { + this._hasRestedLastFrame = this._hasRested + const dampingFactor = + this._state === ACTION.NONE ? this.dampingFactor : this.draggingDampingFactor + const lerpRatio = Math.min(dampingFactor * delta * 60, 1) + const deltaTheta = this._sphericalEnd.theta - this._spherical.theta + const deltaPhi = this._sphericalEnd.phi - this._spherical.phi + const deltaRadius = this._sphericalEnd.radius - this._spherical.radius + const deltaTarget = _deltaTarget.subVectors(this._targetEnd, this._target) + const deltaOffset = _deltaOffset.subVectors(this._focalOffsetEnd, this._focalOffset) + if ( + !approxZero(deltaTheta) || + !approxZero(deltaPhi) || + !approxZero(deltaRadius) || + !approxZero(deltaTarget.x) || + !approxZero(deltaTarget.y) || + !approxZero(deltaTarget.z) || + !approxZero(deltaOffset.x) || + !approxZero(deltaOffset.y) || + !approxZero(deltaOffset.z) + ) { + this._spherical.set( + this._spherical.radius + deltaRadius * lerpRatio, + this._spherical.phi + deltaPhi * lerpRatio, + this._spherical.theta + deltaTheta * lerpRatio + ) + this._target.add(deltaTarget.multiplyScalar(lerpRatio)) + this._focalOffset.add(deltaOffset.multiplyScalar(lerpRatio)) + this._needsUpdate = true + } else { + this._spherical.copy(this._sphericalEnd) + this._target.copy(this._targetEnd) + this._focalOffset.copy(this._focalOffsetEnd) + } + if (this._dollyControlAmount !== 0) { + if (isPerspectiveCamera(this._camera)) { + const camera = this._camera + const direction = _v3A + .setFromSpherical(this._sphericalEnd) + .applyQuaternion(this._yAxisUpSpaceInverse) + .normalize() + .negate() + const planeX = _v3B.copy(direction).cross(camera.up).normalize() + if (planeX.lengthSq() === 0) planeX.x = 1.0 + const planeY = _v3C.crossVectors(planeX, direction) + const worldToScreen = + this._sphericalEnd.radius * + Math.tan( + (camera as PerspectiveCamera).getEffectiveFOV() * MathUtils.DEG2RAD * 0.5 + ) + const prevRadius = this._sphericalEnd.radius - this._dollyControlAmount + const lerpRatio = + (prevRadius - this._sphericalEnd.radius) / this._sphericalEnd.radius + const cursor = _v3A + .copy(this._targetEnd) + .add( + planeX.multiplyScalar( + this._dollyControlCoord.x * + worldToScreen * + (camera as PerspectiveCamera).aspect + ) + ) + .add(planeY.multiplyScalar(this._dollyControlCoord.y * worldToScreen)) + this._targetEnd.lerp(cursor, lerpRatio) + this._target.copy(this._targetEnd) + } else if (isOrthographicCamera(this._camera)) { + const camera = this._camera + const worldPosition = _v3A + .set( + this._dollyControlCoord.x, + this._dollyControlCoord.y, + (camera.near + camera.far) / (camera.near - camera.far) + ) + .unproject(camera) + const quaternion = _v3B.set(0, 0, -1).applyQuaternion(camera.quaternion) + const divisor = quaternion.dot(camera.up) + const distance = approxZero(divisor) + ? -worldPosition.dot(camera.up) + : -worldPosition.dot(camera.up) / divisor + const cursor = _v3C.copy(worldPosition).add(quaternion.multiplyScalar(distance)) + this._targetEnd.lerp(cursor, 1 - camera.zoom / this._dollyControlAmount) + this._target.copy(this._targetEnd) + } + this._dollyControlAmount = 0 + } + const maxDistance = this._collisionTest() + this._spherical.radius = Math.min(this._spherical.radius, maxDistance) + this._spherical.makeSafe() + this._camera.position + .setFromSpherical(this._spherical) + .applyQuaternion(this._yAxisUpSpaceInverse) + .add(this._target) + this._camera.lookAt(this._target) + const affectOffset = + !approxZero(this._focalOffset.x) || + !approxZero(this._focalOffset.y) || + !approxZero(this._focalOffset.z) + if (affectOffset) { + this._camera.updateMatrix() + _xColumn.setFromMatrixColumn(this._camera.matrix, 0) + _yColumn.setFromMatrixColumn(this._camera.matrix, 1) + _zColumn.setFromMatrixColumn(this._camera.matrix, 2) + _xColumn.multiplyScalar(this._focalOffset.x) + _yColumn.multiplyScalar(-this._focalOffset.y) + _zColumn.multiplyScalar(this._focalOffset.z) + _v3A.copy(_xColumn).add(_yColumn).add(_zColumn) + this._camera.position.add(_v3A) + } + if (this._boundaryEnclosesCamera) { + this._encloseToBoundary( + this._camera.position.copy(this._target), + _v3A + .setFromSpherical(this._spherical) + .applyQuaternion(this._yAxisUpSpaceInverse), + 1.0 + ) + } + const zoomDelta = this._zoomEnd - this._zoom + this._zoom += zoomDelta * lerpRatio + if (this._camera.zoom !== this._zoom) { + if (approxZero(zoomDelta)) this._zoom = this._zoomEnd + this._camera.zoom = this._zoom + this._camera.updateProjectionMatrix() + this._updateNearPlaneCorners() + this._needsUpdate = true + } + const updated = this._needsUpdate + if (updated && !this._updatedLastTime) { + this._hasRested = false + this.dispatchEvent({ type: 'wake' }) + this.dispatchEvent({ type: 'update' }) + } else if (updated) { + this.dispatchEvent({ type: 'update' }) + if ( + approxZero(deltaTheta, this.restThreshold) && + approxZero(deltaPhi, this.restThreshold) && + approxZero(deltaRadius, this.restThreshold) && + approxZero(deltaTarget.x, this.restThreshold) && + approxZero(deltaTarget.y, this.restThreshold) && + approxZero(deltaTarget.z, this.restThreshold) && + approxZero(deltaOffset.x, this.restThreshold) && + approxZero(deltaOffset.y, this.restThreshold) && + approxZero(deltaOffset.z, this.restThreshold) && + !this._hasRested && + !this._isTrucking + ) { + this._hasRested = true + this.dispatchEvent({ type: 'rest' }) + } + } else if (!updated && this._updatedLastTime) { + this.dispatchEvent({ type: 'sleep' }) + } + if (this._didDolly) { + if ( + approxZero(deltaTheta, this.restThreshold) && + approxZero(deltaPhi, this.restThreshold) && + approxZero(deltaRadius, this.restThreshold) && + approxZero(deltaTarget.x, this.restThreshold) && + approxZero(deltaTarget.y, this.restThreshold) && + approxZero(deltaTarget.z, this.restThreshold) && + approxZero(deltaOffset.x, this.restThreshold) && + approxZero(deltaOffset.y, this.restThreshold) && + approxZero(deltaOffset.z, this.restThreshold) && + !this._isTrucking + ) { + this.dispatchEvent({ type: 'rest' }) + this._didDolly = false + } + } + this._updatedLastTime = updated + this._needsUpdate = false + return updated && !this._hasRested + } +} diff --git a/packages/viewer/src/modules/pipeline/Pipeline.ts b/packages/viewer/src/modules/pipeline/Pipeline.ts index ed637f986..3810414cb 100644 --- a/packages/viewer/src/modules/pipeline/Pipeline.ts +++ b/packages/viewer/src/modules/pipeline/Pipeline.ts @@ -305,7 +305,7 @@ export class Pipeline { this._renderer.clear(true) if (this.renderType === RenderType.NORMAL) { this.composer.render() - return true + return false } else { console.warn('Rendering accumulation frame -> ', this.accumulationFrame) this.composer.render() @@ -327,7 +327,6 @@ export class Pipeline { } this.renderType = RenderType.ACCUMULATION this.accumulationFrame = 0 - this.depthPass.enabled = true // this.normalsPass.enabled = false this.dynamicAoPass.enabled = false @@ -336,7 +335,6 @@ export class Pipeline { this.staticAoPass.enabled = true this.applySaoPass.setTexture('tDiffuse', this.staticAoPass.outputTexture) this.applySaoPass.setTexture('tDiffuseInterp', this.dynamicAoPass.outputTexture) - this.applySaoPass.setRenderType(this.renderType) console.warn('Starting stationary') } @@ -346,12 +344,10 @@ export class Pipeline { if (this.renderType === RenderType.NORMAL) return this.accumulationFrame = 0 this.renderType = RenderType.NORMAL - this.staticAoPass.enabled = false this.applySaoPass.enabled = true this.dynamicAoPass.enabled = true this.applySaoPass.setTexture('tDiffuse', this.dynamicAoPass.outputTexture) - this.applySaoPass.setRenderType(this.renderType) console.warn('Ending stationary') } From ffd8b4d894041a6c9e501517e3acb66814f1cf5d Mon Sep 17 00:00:00 2001 From: AlexandruPopovici Date: Tue, 25 Oct 2022 19:07:55 +0300 Subject: [PATCH 23/54] Added switching between perspective and linear depth when going from dynamic AO to progressive AO, so we don't have to integrate linear depth buffer sampling in the already existing features --- .../shaders/speckle-static-ao-generate-frag.ts | 2 +- packages/viewer/src/modules/pipeline/DepthPass.ts | 14 +++++++++++++- .../viewer/src/modules/pipeline/DynamicAOPass.ts | 2 +- packages/viewer/src/modules/pipeline/Pipeline.ts | 4 +++- 4 files changed, 18 insertions(+), 4 deletions(-) diff --git a/packages/viewer/src/modules/materials/shaders/speckle-static-ao-generate-frag.ts b/packages/viewer/src/modules/materials/shaders/speckle-static-ao-generate-frag.ts index af9e0ffbc..cfeed5b7e 100644 --- a/packages/viewer/src/modules/materials/shaders/speckle-static-ao-generate-frag.ts +++ b/packages/viewer/src/modules/materials/shaders/speckle-static-ao-generate-frag.ts @@ -251,7 +251,7 @@ export const speckleStaticAoGenerateFrag = /* glsl */ ` vec4 samplePointNDC = cameraProjectionMatrix * vec4( samplePoint, 1.0 ); // project point and calculate NDC samplePointNDC /= samplePointNDC.w; vec2 samplePointUv = samplePointNDC.xy * 0.5 + 0.5; // compute uv coordinates - float realDepth = unpackRGBAToDepth( texture2D( tDepth, samplePointUv ) );//getLinearDepth( samplePointUv ); // get linear depth from depth texture + float realDepth = getLinearDepth( samplePointUv ); // get linear depth from depth texture float sampleDepth = viewZToOrthographicDepth( samplePoint.z + bias, cameraNear, cameraFar ); // compute linear depth of the sample view Z value float delta = sampleDepth - realDepth; if ( delta > minDistance && delta < maxDistance ) { // if fragment is before sample point, increase occlusion diff --git a/packages/viewer/src/modules/pipeline/DepthPass.ts b/packages/viewer/src/modules/pipeline/DepthPass.ts index 63ed6e5e4..d852ba3a1 100644 --- a/packages/viewer/src/modules/pipeline/DepthPass.ts +++ b/packages/viewer/src/modules/pipeline/DepthPass.ts @@ -15,6 +15,11 @@ import { Pass } from 'three/examples/jsm/postprocessing/Pass' import SpeckleDepthMaterial from '../materials/SpeckleDepthMaterial' import { SpecklePass } from './SpecklePass' +export enum DepthType { + PERSPECTIVE_DEPTH, + LINEAR_DEPTH +} + export class DepthPass extends Pass implements SpecklePass { private renderTarget: WebGLRenderTarget private depthMaterial: SpeckleDepthMaterial = null @@ -34,6 +39,13 @@ export class DepthPass extends Pass implements SpecklePass { return this.renderTarget.texture } + public set depthType(value: DepthType) { + if (value === DepthType.LINEAR_DEPTH) + this.depthMaterial.defines['LINEAR_DEPTH'] = ' ' + else delete this.depthMaterial.defines['LINEAR_DEPTH'] + this.depthMaterial.needsUpdate = true + } + constructor() { super() @@ -49,7 +61,7 @@ export class DepthPass extends Pass implements SpecklePass { { depthPacking: RGBADepthPacking }, - ['USE_RTE', 'ALPHATEST_REJECTION', 'LINEAR_DEPTH'] + ['USE_RTE', 'ALPHATEST_REJECTION'] ) this.depthMaterial.blending = NoBlending this.depthMaterial.side = DoubleSide diff --git a/packages/viewer/src/modules/pipeline/DynamicAOPass.ts b/packages/viewer/src/modules/pipeline/DynamicAOPass.ts index 35a48799d..b9a124b0e 100644 --- a/packages/viewer/src/modules/pipeline/DynamicAOPass.ts +++ b/packages/viewer/src/modules/pipeline/DynamicAOPass.ts @@ -69,7 +69,7 @@ export class DynamicSAOPass extends Pass implements SpecklePass { private blurIntermediateRenderTarget: WebGLRenderTarget = null private fsQuad: FullScreenQuad = null private _outputType: DynamicAOOutputType = DynamicAOOutputType.AO_BLURRED - private outputScale = 0.5 + private outputScale = 1 private prevStdDev: number private prevNumSamples: number diff --git a/packages/viewer/src/modules/pipeline/Pipeline.ts b/packages/viewer/src/modules/pipeline/Pipeline.ts index 3810414cb..6db55779d 100644 --- a/packages/viewer/src/modules/pipeline/Pipeline.ts +++ b/packages/viewer/src/modules/pipeline/Pipeline.ts @@ -7,7 +7,7 @@ import Batcher from '../batching/Batcher' import SpeckleRenderer from '../SpeckleRenderer' import { ApplySAOPass } from './ApplyAOPass' import { CopyOutputPass } from './CopyOutputPass' -import { DepthPass } from './DepthPass' +import { DepthPass, DepthType } from './DepthPass' import { NormalsPass } from './NormalsPass' import { DefaultDynamicAOPassParams, @@ -328,6 +328,7 @@ export class Pipeline { this.renderType = RenderType.ACCUMULATION this.accumulationFrame = 0 this.depthPass.enabled = true + this.depthPass.depthType = DepthType.LINEAR_DEPTH // this.normalsPass.enabled = false this.dynamicAoPass.enabled = false this.renderPass.enabled = true @@ -344,6 +345,7 @@ export class Pipeline { if (this.renderType === RenderType.NORMAL) return this.accumulationFrame = 0 this.renderType = RenderType.NORMAL + this.depthPass.depthType = DepthType.PERSPECTIVE_DEPTH this.staticAoPass.enabled = false this.applySaoPass.enabled = true this.dynamicAoPass.enabled = true From 6c35f6c50a1132ae72dc50ac0ee63210b7e09c60 Mon Sep 17 00:00:00 2001 From: AlexandruPopovici Date: Tue, 25 Oct 2022 19:24:04 +0300 Subject: [PATCH 24/54] Disabled normal texture rendering for dynamic AO --- packages/viewer-sandbox/src/main.ts | 4 ++-- packages/viewer/src/modules/pipeline/DynamicAOPass.ts | 2 +- packages/viewer/src/modules/pipeline/Pipeline.ts | 9 +++++---- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/packages/viewer-sandbox/src/main.ts b/packages/viewer-sandbox/src/main.ts index 005d3fddb..00a7cdb84 100644 --- a/packages/viewer-sandbox/src/main.ts +++ b/packages/viewer-sandbox/src/main.ts @@ -82,11 +82,11 @@ sandbox.makeFilteringUI() await sandbox.loadUrl( // 'https://speckle.xyz/streams/da9e320dad/commits/5388ef24b8?c=%5B-7.66134,10.82932,6.41935,-0.07739,-13.88552,1.8697,0,1%5D' // Revit sample house (good for bim-like stuff with many display meshes) - 'https://speckle.xyz/streams/da9e320dad/commits/5388ef24b8' + // 'https://speckle.xyz/streams/da9e320dad/commits/5388ef24b8' // 'Super' heavy revit shit // 'https://speckle.xyz/streams/e6f9156405/commits/0694d53bb5' // IFC building (good for a tree based structure) - // 'https://latest.speckle.dev/streams/92b620fb17/commits/2ebd336223' + 'https://latest.speckle.dev/streams/92b620fb17/commits/2ebd336223' // IFC story, a subtree of the above // 'https://latest.speckle.dev/streams/92b620fb17/objects/8247bbc53865b0e0cb5ee4e252e66216' // Small scale lines diff --git a/packages/viewer/src/modules/pipeline/DynamicAOPass.ts b/packages/viewer/src/modules/pipeline/DynamicAOPass.ts index b9a124b0e..35a48799d 100644 --- a/packages/viewer/src/modules/pipeline/DynamicAOPass.ts +++ b/packages/viewer/src/modules/pipeline/DynamicAOPass.ts @@ -69,7 +69,7 @@ export class DynamicSAOPass extends Pass implements SpecklePass { private blurIntermediateRenderTarget: WebGLRenderTarget = null private fsQuad: FullScreenQuad = null private _outputType: DynamicAOOutputType = DynamicAOOutputType.AO_BLURRED - private outputScale = 1 + private outputScale = 0.5 private prevStdDev: number private prevNumSamples: number diff --git a/packages/viewer/src/modules/pipeline/Pipeline.ts b/packages/viewer/src/modules/pipeline/Pipeline.ts index 6db55779d..2ad5b047c 100644 --- a/packages/viewer/src/modules/pipeline/Pipeline.ts +++ b/packages/viewer/src/modules/pipeline/Pipeline.ts @@ -241,10 +241,11 @@ export class Pipeline { private getDefaultPipeline(): Array { this.renderPass.renderToScreen = true - this.normalsPass.enabled = true - // this._pipelineOptions.dynamicAoParams.normalsType === NormalsType.DEFAULT - // ? true - // : false + this.normalsPass.enabled = + this._pipelineOptions.dynamicAoParams.normalsType === NormalsType.DEFAULT + ? true + : false + this.dynamicAoPass.setOutputType( this._pipelineOptions.dynamicAoParams.blurEnabled ? DynamicAOOutputType.AO_BLURRED From 3d770f754c1efbe6590f35a3a1308158a0467574 Mon Sep 17 00:00:00 2001 From: AlexandruPopovici Date: Tue, 25 Oct 2022 22:00:48 +0300 Subject: [PATCH 25/54] Some fixes and added dynamic AO back in at half res --- packages/viewer-sandbox/src/Sandbox.ts | 4 ++-- .../viewer/src/modules/SpeckleRenderer.ts | 6 ++++++ .../modules/objects/SpeckleCameraControls.ts | 11 +++++++++-- .../src/modules/pipeline/DynamicAOPass.ts | 4 ++-- .../viewer/src/modules/pipeline/Pipeline.ts | 19 ++++++++++++++++--- 5 files changed, 35 insertions(+), 9 deletions(-) diff --git a/packages/viewer-sandbox/src/Sandbox.ts b/packages/viewer-sandbox/src/Sandbox.ts index 3f8f78c7b..f351f5b64 100644 --- a/packages/viewer-sandbox/src/Sandbox.ts +++ b/packages/viewer-sandbox/src/Sandbox.ts @@ -37,11 +37,11 @@ export default class Sandbox { dynamicAoParams: { intensity: 1.25, scale: 0, - kernelRadius: 10, + kernelRadius: 5, bias: 0.15, normalsType: 2, blurEnabled: true, - blurRadius: 1, + blurRadius: 2, blurStdDev: 4, blurDepthCutoff: 0.0007 }, diff --git a/packages/viewer/src/modules/SpeckleRenderer.ts b/packages/viewer/src/modules/SpeckleRenderer.ts index 867899ca8..305b0da50 100644 --- a/packages/viewer/src/modules/SpeckleRenderer.ts +++ b/packages/viewer/src/modules/SpeckleRenderer.ts @@ -342,6 +342,11 @@ export default class SpeckleRenderer { this.pipeline.update(this) } + public resetPipeline() { + this._needsRender = true + this.pipeline.reset() + } + public render(): void { if (this._needsRender) { this.batcher.render(this.renderer) @@ -405,6 +410,7 @@ export default class SpeckleRenderer { this.updateDirectLights() this.updateHelpers() + this.resetPipeline() } public removeRenderTree(subtreeId: string) { diff --git a/packages/viewer/src/modules/objects/SpeckleCameraControls.ts b/packages/viewer/src/modules/objects/SpeckleCameraControls.ts index c7467322b..7d883528f 100644 --- a/packages/viewer/src/modules/objects/SpeckleCameraControls.ts +++ b/packages/viewer/src/modules/objects/SpeckleCameraControls.ts @@ -41,6 +41,7 @@ let _zColumn export class SpeckleCameraControls extends CameraControls { private _didDolly = false + private _didDollyLastFrame = false public _isTrucking = false private _hasRestedLastFrame = false static install() { @@ -258,7 +259,7 @@ export class SpeckleCameraControls extends CameraControls { } else if (!updated && this._updatedLastTime) { this.dispatchEvent({ type: 'sleep' }) } - if (this._didDolly) { + if (this._didDollyLastFrame) { if ( approxZero(deltaTheta, this.restThreshold) && approxZero(deltaPhi, this.restThreshold) && @@ -272,9 +273,15 @@ export class SpeckleCameraControls extends CameraControls { !this._isTrucking ) { this.dispatchEvent({ type: 'rest' }) - this._didDolly = false + this._didDollyLastFrame = false } } + + if (this._didDolly) { + this._didDolly = false + this._didDollyLastFrame = true + } + this._updatedLastTime = updated this._needsUpdate = false return updated && !this._hasRested diff --git a/packages/viewer/src/modules/pipeline/DynamicAOPass.ts b/packages/viewer/src/modules/pipeline/DynamicAOPass.ts index 35a48799d..8c030bb34 100644 --- a/packages/viewer/src/modules/pipeline/DynamicAOPass.ts +++ b/packages/viewer/src/modules/pipeline/DynamicAOPass.ts @@ -50,11 +50,11 @@ export interface DynamicAOPassParams { export const DefaultDynamicAOPassParams = { intensity: 1.25, scale: 0, - kernelRadius: 10, + kernelRadius: 5, bias: 0.15, normalsType: NormalsType.ACCURATE, blurEnabled: true, - blurRadius: 1, + blurRadius: 4, blurStdDev: 4, blurDepthCutoff: 0.0007 } diff --git a/packages/viewer/src/modules/pipeline/Pipeline.ts b/packages/viewer/src/modules/pipeline/Pipeline.ts index 2ad5b047c..fba729c31 100644 --- a/packages/viewer/src/modules/pipeline/Pipeline.ts +++ b/packages/viewer/src/modules/pipeline/Pipeline.ts @@ -66,6 +66,7 @@ export class Pipeline { private _batcher: Batcher = null private _pipelineOptions: PipelineOptions = Object.assign({}, DefaultPipelineOptions) private _needsProgressive = false + private _resetFrame = false private composer: EffectComposer = null private depthPass: DepthPass = null @@ -151,6 +152,7 @@ export class Pipeline { ? true : false this.dynamicAoPass.enabled = true + this.depthPass.depthType = DepthType.PERSPECTIVE_DEPTH this.copyOutputPass.setTexture('tDiffuse', this.dynamicAoPass.outputTexture) this.copyOutputPass.setOutputType(PipelineOutputType.COLOR) this.dynamicAoPass.setOutputType(DynamicAOOutputType.AO) @@ -167,6 +169,7 @@ export class Pipeline { ? true : false this.dynamicAoPass.enabled = true + this.depthPass.depthType = DepthType.PERSPECTIVE_DEPTH this.copyOutputPass.setTexture('tDiffuse', this.dynamicAoPass.outputTexture) this.copyOutputPass.setOutputType(PipelineOutputType.COLOR) this.dynamicAoPass.setOutputType(DynamicAOOutputType.AO_BLURRED) @@ -179,7 +182,7 @@ export class Pipeline { pipeline.push(this.dynamicAoPass) pipeline.push(this.staticAoPass) pipeline.push(this.copyOutputPass) - this.normalsPass.enabled + this.depthPass.depthType = DepthType.LINEAR_DEPTH this.copyOutputPass.setTexture('tDiffuse', this.staticAoPass.outputTexture) this.copyOutputPass.setOutputType(PipelineOutputType.COLOR) this.needsProgressive = true @@ -288,6 +291,11 @@ export class Pipeline { this.depthPass.setClippingPlanes(planes) } + public reset() { + this._resetFrame = true + this.onStationaryEnd() + } + public update(renderer: SpeckleRenderer) { this.depthPass.update(renderer.scene, renderer.camera) this.dynamicAoPass.update(renderer.scene, renderer.camera) @@ -306,7 +314,12 @@ export class Pipeline { this._renderer.clear(true) if (this.renderType === RenderType.NORMAL) { this.composer.render() - return false + const ret = false || this._resetFrame + if (this._resetFrame) { + this._resetFrame = false + this.onStationaryBegin() + } + return ret } else { console.warn('Rendering accumulation frame -> ', this.accumulationFrame) this.composer.render() @@ -330,7 +343,7 @@ export class Pipeline { this.accumulationFrame = 0 this.depthPass.enabled = true this.depthPass.depthType = DepthType.LINEAR_DEPTH - // this.normalsPass.enabled = false + this.normalsPass.enabled = false this.dynamicAoPass.enabled = false this.renderPass.enabled = true this.applySaoPass.enabled = true From f18f7b5372760c31e0585e3bb7201429b168df68 Mon Sep 17 00:00:00 2001 From: AlexandruPopovici Date: Tue, 25 Oct 2022 22:38:29 +0300 Subject: [PATCH 26/54] Fixed an issue with converging progressive pipeline when using a trackpad --- packages/viewer-sandbox/src/Sandbox.ts | 2 +- .../modules/objects/SpeckleCameraControls.ts | 38 ++++++++++++++++++- 2 files changed, 37 insertions(+), 3 deletions(-) diff --git a/packages/viewer-sandbox/src/Sandbox.ts b/packages/viewer-sandbox/src/Sandbox.ts index e670bbc28..b8db31579 100644 --- a/packages/viewer-sandbox/src/Sandbox.ts +++ b/packages/viewer-sandbox/src/Sandbox.ts @@ -529,7 +529,7 @@ export default class Sandbox { this.viewer.requestRender() }) - staticAoFolder + staticAoFolder .addInput(Sandbox.pipelineParams.staticAoParams, 'bias', { min: -1, max: 1, diff --git a/packages/viewer/src/modules/objects/SpeckleCameraControls.ts b/packages/viewer/src/modules/objects/SpeckleCameraControls.ts index 7d883528f..82823ea02 100644 --- a/packages/viewer/src/modules/objects/SpeckleCameraControls.ts +++ b/packages/viewer/src/modules/objects/SpeckleCameraControls.ts @@ -63,13 +63,46 @@ export class SpeckleCameraControls extends CameraControls { this._isTrucking = value } + protected _dollyInternal = (delta: number, x: number, y: number): void => { + const dollyScale = Math.pow(0.95, -delta * this.dollySpeed) + const distance = this._sphericalEnd.radius * dollyScale + const prevRadius = this._sphericalEnd.radius + const signedPrevRadius = prevRadius * (delta >= 0 ? -1 : 1) + + this.dollyTo(distance) + + if ( + this.infinityDolly && + (distance < this.minDistance || this.maxDistance === this.minDistance) + ) { + this._camera.getWorldDirection(_v3A) + this._targetEnd.add(_v3A.normalize().multiplyScalar(signedPrevRadius)) + this._target.add(_v3A.normalize().multiplyScalar(signedPrevRadius)) + } + + if (this.dollyToCursor) { + this._dollyControlAmount += this._sphericalEnd.radius - prevRadius + + if ( + this.infinityDolly && + (distance < this.minDistance || this.maxDistance === this.minDistance) + ) { + this._dollyControlAmount -= signedPrevRadius + } + + this._dollyControlCoord.set(x, y) + } + + return + } + /** * Dolly in/out camera position to given distance. * @param distance Distance of dolly. * @param enableTransition Whether to move smoothly or immediately. * @category Methods */ - dollyTo(distance: number, enableTransition = false): Promise { + dollyTo(distance: number, enableTransition = true): Promise { const lastRadius = this._sphericalEnd.radius const newRadius = MathUtils.clamp(distance, this.minDistance, this.maxDistance) const hasCollider = this.colliderMeshes.length >= 1 @@ -111,6 +144,7 @@ export class SpeckleCameraControls extends CameraControls { this._hasRestedLastFrame = this._hasRested const dampingFactor = this._state === ACTION.NONE ? this.dampingFactor : this.draggingDampingFactor + const dampingFactorDolly = 0.9 const lerpRatio = Math.min(dampingFactor * delta * 60, 1) const deltaTheta = this._sphericalEnd.theta - this._spherical.theta const deltaPhi = this._sphericalEnd.phi - this._spherical.phi @@ -129,7 +163,7 @@ export class SpeckleCameraControls extends CameraControls { !approxZero(deltaOffset.z) ) { this._spherical.set( - this._spherical.radius + deltaRadius * lerpRatio, + this._spherical.radius + deltaRadius * dampingFactorDolly, this._spherical.phi + deltaPhi * lerpRatio, this._spherical.theta + deltaTheta * lerpRatio ) From 72307927ccf337864af1bac15eee8487bd4536a8 Mon Sep 17 00:00:00 2001 From: AlexandruPopovici Date: Thu, 27 Oct 2022 09:34:38 +0300 Subject: [PATCH 27/54] Implemented relative dynamic progressive AO kernel size --- packages/viewer-sandbox/src/Sandbox.ts | 4 ++-- packages/viewer-sandbox/src/main.ts | 4 ++-- packages/viewer/src/modules/SpeckleRenderer.ts | 1 - .../materials/shaders/speckle-apply-ao-frag.ts | 3 ++- .../shaders/speckle-static-ao-generate-frag.ts | 15 +++++++++++++-- .../src/modules/pipeline/StaticAOPass.ts | 18 +++++++++++------- 6 files changed, 30 insertions(+), 15 deletions(-) diff --git a/packages/viewer-sandbox/src/Sandbox.ts b/packages/viewer-sandbox/src/Sandbox.ts index b8db31579..76732ded9 100644 --- a/packages/viewer-sandbox/src/Sandbox.ts +++ b/packages/viewer-sandbox/src/Sandbox.ts @@ -48,7 +48,7 @@ export default class Sandbox { staticAoEnabled: true, staticAoParams: { intensity: 0.8, - kernelRadius: 0.35, // World space + kernelRadius: 20, // Screen space kernelSize: 16, bias: 0.01, minDistance: 0, @@ -522,7 +522,7 @@ export default class Sandbox { staticAoFolder .addInput(Sandbox.pipelineParams.staticAoParams, 'kernelRadius', { min: 0, - max: 100 + max: 1000 }) .on('change', () => { this.viewer.getRenderer().pipelineOptions = Sandbox.pipelineParams diff --git a/packages/viewer-sandbox/src/main.ts b/packages/viewer-sandbox/src/main.ts index 00a7cdb84..01918e9c3 100644 --- a/packages/viewer-sandbox/src/main.ts +++ b/packages/viewer-sandbox/src/main.ts @@ -86,7 +86,7 @@ await sandbox.loadUrl( // 'Super' heavy revit shit // 'https://speckle.xyz/streams/e6f9156405/commits/0694d53bb5' // IFC building (good for a tree based structure) - 'https://latest.speckle.dev/streams/92b620fb17/commits/2ebd336223' + // 'https://latest.speckle.dev/streams/92b620fb17/commits/2ebd336223' // IFC story, a subtree of the above // 'https://latest.speckle.dev/streams/92b620fb17/objects/8247bbc53865b0e0cb5ee4e252e66216' // Small scale lines @@ -97,7 +97,7 @@ await sandbox.loadUrl( // AutoCAD // 'https://latest.speckle.dev/streams/3ed8357f29/commits/d10f2af1ce' //Blizzard world - // 'https://latest.speckle.dev/streams/0c6ad366c4/commits/aa1c393aec' + 'https://latest.speckle.dev/streams/0c6ad366c4/commits/aa1c393aec' //Car // 'https://latest.speckle.dev/streams/17d2e25a97/commits/6b6cf3d43e' // Jonathon's diff --git a/packages/viewer/src/modules/SpeckleRenderer.ts b/packages/viewer/src/modules/SpeckleRenderer.ts index 305b0da50..2a058dcab 100644 --- a/packages/viewer/src/modules/SpeckleRenderer.ts +++ b/packages/viewer/src/modules/SpeckleRenderer.ts @@ -312,7 +312,6 @@ export default class SpeckleRenderer { this.viewer.cameraHandler.activeCam.camera.far = d this.viewer.cameraHandler.activeCam.camera.updateProjectionMatrix() this.viewer.cameraHandler.camera.updateProjectionMatrix() - // const currentAzimuth = this.viewer.cameraHandler.controls.azimuthAngle // const currentPolar = this.viewer.cameraHandler.controls.polarAngle // const currentPosition = this.viewer.cameraHandler.activeCam.camera.position diff --git a/packages/viewer/src/modules/materials/shaders/speckle-apply-ao-frag.ts b/packages/viewer/src/modules/materials/shaders/speckle-apply-ao-frag.ts index c8e2b2211..07603cd42 100644 --- a/packages/viewer/src/modules/materials/shaders/speckle-apply-ao-frag.ts +++ b/packages/viewer/src/modules/materials/shaders/speckle-apply-ao-frag.ts @@ -9,8 +9,9 @@ export const speckleApplyAoFrag = ` void main() { vec3 currentSample = texture2D( tDiffuse, vUv ).rgb; - vec3 interpSample = texture2D( tDiffuseInterp, vUv ).rgb; + #if ACCUMULATE == 1 + vec3 interpSample = texture2D( tDiffuseInterp, vUv ).rgb; gl_FragColor.rgb = mix(interpSample, currentSample, frameIndex/float(NUM_FRAMES)); #else gl_FragColor.rgb = currentSample; diff --git a/packages/viewer/src/modules/materials/shaders/speckle-static-ao-generate-frag.ts b/packages/viewer/src/modules/materials/shaders/speckle-static-ao-generate-frag.ts index cfeed5b7e..cf3d48dc4 100644 --- a/packages/viewer/src/modules/materials/shaders/speckle-static-ao-generate-frag.ts +++ b/packages/viewer/src/modules/materials/shaders/speckle-static-ao-generate-frag.ts @@ -16,6 +16,7 @@ export const speckleStaticAoGenerateFrag = /* glsl */ ` uniform float kernelRadius; uniform float minResolution; uniform float frameIndex; + uniform float tanFov; #define AO_ESTIMATOR 1 // #define KERNEL_SIZE 16 @@ -191,7 +192,16 @@ export const speckleStaticAoGenerateFrag = /* glsl */ ` // moving costly divides into consts const float INV_NUM_SAMPLES = 1.0 / float( NUM_SAMPLES ); const float offset = PI2 / float(NUM_FRAMES); - + + float computeKernelSize(float d, float r) { + // Apparently this is wrong + // return (r * tan(fov) * d) / (size.y * 0.5); + // And this is correct + float rp = r / (size.y * 0.5); + return sqrt((rp*rp*tanFov*tanFov*d*d)/(1. + rp*rp*tanFov*tanFov)); + // return r; + } + float getAmbientOcclusion( const in vec3 centerViewPosition, in float centerDepth ) { #if AO_ESTIMATOR == 0 // precompute some variables require in getOcclusion. @@ -245,9 +255,10 @@ export const speckleStaticAoGenerateFrag = /* glsl */ ` vec3 bitangent = cross( viewNormal, tangent ); mat3 kernelMatrix = mat3( tangent, bitangent, viewNormal ); float occlusion = 0.0; + float kernelSize_ws = computeKernelSize(-viewPosition.z, kernelRadius); for ( int i = 0; i < KERNEL_SIZE; i ++ ) { vec3 sampleVector = kernelMatrix * kernel[ i ]; // reorient sample vector in view space - vec3 samplePoint = viewPosition + ( sampleVector * kernelRadius ); // calculate sample point + vec3 samplePoint = viewPosition + ( sampleVector * kernelSize_ws ); // calculate sample point vec4 samplePointNDC = cameraProjectionMatrix * vec4( samplePoint, 1.0 ); // project point and calculate NDC samplePointNDC /= samplePointNDC.w; vec2 samplePointUv = samplePointNDC.xy * 0.5 + 0.5; // compute uv coordinates diff --git a/packages/viewer/src/modules/pipeline/StaticAOPass.ts b/packages/viewer/src/modules/pipeline/StaticAOPass.ts index 71cd070f4..35e3a3205 100644 --- a/packages/viewer/src/modules/pipeline/StaticAOPass.ts +++ b/packages/viewer/src/modules/pipeline/StaticAOPass.ts @@ -40,17 +40,17 @@ import { Pipeline } from './Pipeline' */ export interface StaticAoPassParams { - intensity: number - kernelRadius: number - kernelSize: number - bias: number - minDistance: number - maxDistance: number + intensity?: number + kernelRadius?: number + kernelSize?: number + bias?: number + minDistance?: number + maxDistance?: number } export const DefaultStaticAoPassParams = { intensity: 0.8, - kernelRadius: 0.35, // World space + kernelRadius: 20, // Screen space kernelSize: 16, bias: 0.01, minDistance: 0, @@ -107,6 +107,7 @@ export class StaticAOPass extends Pass implements SpeckleProgressivePass { cameraFar: { value: 100 }, cameraProjectionMatrix: { value: new Matrix4() }, cameraInverseProjectionMatrix: { value: new Matrix4() }, + tanFov: { value: 0 }, scale: { value: 1.0 }, intensity: { value: 1 }, @@ -181,6 +182,9 @@ export class StaticAOPass extends Pass implements SpeckleProgressivePass { this.aoMaterial.uniforms['cameraProjectionMatrix'].value.copy( camera.projectionMatrix ) + const fov = (((camera as PerspectiveCamera).fov / 2) * Math.PI) / 180.0 + this.aoMaterial.uniforms['tanFov'].value = Math.tan(fov) + if (!this.kernels[this.frameIndex]) { this.generateSampleKernel(this.frameIndex) } From a3c444a102123a2756fabb0ab9469876194acda3 Mon Sep 17 00:00:00 2001 From: Alex Date: Thu, 27 Oct 2022 09:55:44 +0300 Subject: [PATCH 28/54] Fixed compile error --- packages/viewer-sandbox/src/main.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/viewer-sandbox/src/main.ts b/packages/viewer-sandbox/src/main.ts index c0d47b5a4..3b48e5239 100644 --- a/packages/viewer-sandbox/src/main.ts +++ b/packages/viewer-sandbox/src/main.ts @@ -119,5 +119,5 @@ await sandbox.loadUrl( // Arcs //'https://latest.speckle.dev/streams/0c6ad366c4/commits/912d83412e' // Freezers - 'https://speckle.xyz/streams/f0532359ac/commits/98678e2a3d?c=%5B2455.15367,2689.87156,4366.68444,205.422,-149.41199,148.749,0,1%5D' + // 'https://speckle.xyz/streams/f0532359ac/commits/98678e2a3d?c=%5B2455.15367,2689.87156,4366.68444,205.422,-149.41199,148.749,0,1%5D' ) From 3653083ece05920fc1057cfae3e26a9e3364041b Mon Sep 17 00:00:00 2001 From: Alex Date: Thu, 27 Oct 2022 11:55:23 +0300 Subject: [PATCH 29/54] Fixed an issue with macOS and depth texture filtering --- .../modules/materials/shaders/speckle-copy-output-frag.ts | 6 +++--- .../src/modules/materials/shaders/speckle-sao-frag.ts | 3 +-- packages/viewer/src/modules/pipeline/DepthPass.ts | 6 +++++- packages/viewer/src/modules/pipeline/Pipeline.ts | 1 + 4 files changed, 10 insertions(+), 6 deletions(-) diff --git a/packages/viewer/src/modules/materials/shaders/speckle-copy-output-frag.ts b/packages/viewer/src/modules/materials/shaders/speckle-copy-output-frag.ts index f696ba58d..f88de9c84 100644 --- a/packages/viewer/src/modules/materials/shaders/speckle-copy-output-frag.ts +++ b/packages/viewer/src/modules/materials/shaders/speckle-copy-output-frag.ts @@ -21,9 +21,9 @@ export const speckleCopyOutputFrag = ` #if OUTPUT_TYPE == 1 outSample.rgb = vec3(unpackRGBAToDepth(inSample)); #endif - #if OUTPUT_TYPE == 3 - outSample.rgb = unpackRGBToNormal(inSample.rgb); - #endif + // #if OUTPUT_TYPE == 3 + // outSample.rgb = unpackRGBToNormal(inSample.rgb); + // #endif gl_FragColor.rgb = outSample; gl_FragColor.a = 1.; diff --git a/packages/viewer/src/modules/materials/shaders/speckle-sao-frag.ts b/packages/viewer/src/modules/materials/shaders/speckle-sao-frag.ts index 66293cd88..787c83d6f 100644 --- a/packages/viewer/src/modules/materials/shaders/speckle-sao-frag.ts +++ b/packages/viewer/src/modules/materials/shaders/speckle-sao-frag.ts @@ -219,7 +219,7 @@ export const speckleSaoFrag = /* glsl */ ` #else normal = normalize( cross( dFdx( viewPosition ), dFdy( viewPosition ) ) ); #endif - gl_FragColor.rgb = packNormalToRGB(viewNormalImproved(vUv, viewPosition)); + gl_FragColor.rgb = packNormalToRGB(normal); gl_FragColor.a = 1.; return; #endif @@ -227,5 +227,4 @@ export const speckleSaoFrag = /* glsl */ ` float ambientOcclusion = getAmbientOcclusion( viewPosition, centerDepth ); gl_FragColor = getDefaultColor( vUv ); gl_FragColor.xyz *= 1. - ambientOcclusion; - // gl_FragColor.xyz = depth_cross(vUv, viewPosition) * 0.5 + 0.5; }` diff --git a/packages/viewer/src/modules/pipeline/DepthPass.ts b/packages/viewer/src/modules/pipeline/DepthPass.ts index d852ba3a1..906abd145 100644 --- a/packages/viewer/src/modules/pipeline/DepthPass.ts +++ b/packages/viewer/src/modules/pipeline/DepthPass.ts @@ -2,6 +2,7 @@ import { Camera, Color, DoubleSide, + NearestFilter, NoBlending, OrthographicCamera, PerspectiveCamera, @@ -49,7 +50,10 @@ export class DepthPass extends Pass implements SpecklePass { constructor() { super() - this.renderTarget = new WebGLRenderTarget(256, 256) + this.renderTarget = new WebGLRenderTarget(256, 256, { + minFilter: NearestFilter, + magFilter: NearestFilter + }) /** On Chromium, on MacOS the 16 bit depth render buffer appears broken. * We're not really using a stencil buffer at all, we're just forcing * three.js to use a 24 bit depth render buffer diff --git a/packages/viewer/src/modules/pipeline/Pipeline.ts b/packages/viewer/src/modules/pipeline/Pipeline.ts index cdfe54a78..260baa3b9 100644 --- a/packages/viewer/src/modules/pipeline/Pipeline.ts +++ b/packages/viewer/src/modules/pipeline/Pipeline.ts @@ -136,6 +136,7 @@ export class Pipeline { pipeline.push(this.dynamicAoPass) pipeline.push(this.copyOutputPass) this.dynamicAoPass.enabled = true + this.depthPass.depthType = DepthType.PERSPECTIVE_DEPTH this.dynamicAoPass.setOutputType(DynamicAOOutputType.RECONSTRUCTED_NORMALS) this.copyOutputPass.setTexture('tDiffuse', this.dynamicAoPass.outputTexture) this.copyOutputPass.setOutputType(PipelineOutputType.GEOMETRY_NORMALS) From 639ae9cd2a6a092caab1a8b9575e1e08d6f2fc91 Mon Sep 17 00:00:00 2001 From: Alex Date: Thu, 27 Oct 2022 13:12:08 +0300 Subject: [PATCH 30/54] Added half res depth buffer switching for dynamic AO --- packages/viewer-sandbox/src/Sandbox.ts | 6 ++--- .../viewer/src/modules/pipeline/DepthPass.ts | 25 ++++++++++++++++++- .../src/modules/pipeline/DynamicAOPass.ts | 8 +++--- .../viewer/src/modules/pipeline/Pipeline.ts | 12 +++++++-- 4 files changed, 41 insertions(+), 10 deletions(-) diff --git a/packages/viewer-sandbox/src/Sandbox.ts b/packages/viewer-sandbox/src/Sandbox.ts index 76732ded9..a9c66ffd9 100644 --- a/packages/viewer-sandbox/src/Sandbox.ts +++ b/packages/viewer-sandbox/src/Sandbox.ts @@ -35,10 +35,10 @@ export default class Sandbox { accumulationFrames: 16, dynamicAoEnabled: true, dynamicAoParams: { - intensity: 1.25, + intensity: 1.5, scale: 0, - kernelRadius: 5, - bias: 0.15, + kernelRadius: 10, + bias: 0.2, normalsType: 2, blurEnabled: true, blurRadius: 2, diff --git a/packages/viewer/src/modules/pipeline/DepthPass.ts b/packages/viewer/src/modules/pipeline/DepthPass.ts index 906abd145..7bc94a8ed 100644 --- a/packages/viewer/src/modules/pipeline/DepthPass.ts +++ b/packages/viewer/src/modules/pipeline/DepthPass.ts @@ -21,9 +21,16 @@ export enum DepthType { LINEAR_DEPTH } +export enum DepthSize { + FULL, + HALF +} + export class DepthPass extends Pass implements SpecklePass { private renderTarget: WebGLRenderTarget + private renderTargetHalf: WebGLRenderTarget private depthMaterial: SpeckleDepthMaterial = null + private depthBufferSize: DepthSize = DepthSize.FULL private scene: Scene private camera: Camera @@ -40,6 +47,10 @@ export class DepthPass extends Pass implements SpecklePass { return this.renderTarget.texture } + get outputTextureHalf(): Texture { + return this.renderTargetHalf.texture + } + public set depthType(value: DepthType) { if (value === DepthType.LINEAR_DEPTH) this.depthMaterial.defines['LINEAR_DEPTH'] = ' ' @@ -47,6 +58,10 @@ export class DepthPass extends Pass implements SpecklePass { this.depthMaterial.needsUpdate = true } + public set depthSize(value: DepthSize) { + this.depthBufferSize = value + } + constructor() { super() @@ -54,12 +69,19 @@ export class DepthPass extends Pass implements SpecklePass { minFilter: NearestFilter, magFilter: NearestFilter }) + this.renderTargetHalf = new WebGLRenderTarget(256, 256, { + minFilter: NearestFilter, + magFilter: NearestFilter + }) + /** On Chromium, on MacOS the 16 bit depth render buffer appears broken. * We're not really using a stencil buffer at all, we're just forcing * three.js to use a 24 bit depth render buffer */ this.renderTarget.depthBuffer = true this.renderTarget.stencilBuffer = true + this.renderTargetHalf.depthBuffer = true + this.renderTargetHalf.stencilBuffer = true this.depthMaterial = new SpeckleDepthMaterial( { @@ -96,7 +118,7 @@ export class DepthPass extends Pass implements SpecklePass { const originalClearAlpha = renderer.getClearAlpha() const originalAutoClear = renderer.autoClear - renderer.setRenderTarget(this.renderTarget) + renderer.setRenderTarget(this.depthBufferSize === DepthSize.FULL ? this.renderTarget: this.renderTargetHalf) renderer.autoClear = false renderer.setClearColor(0x000000) @@ -122,5 +144,6 @@ export class DepthPass extends Pass implements SpecklePass { public setSize(width: number, height: number) { this.renderTarget.setSize(width, height) + this.renderTargetHalf.setSize(width * 0.5, height * 0.5) } } diff --git a/packages/viewer/src/modules/pipeline/DynamicAOPass.ts b/packages/viewer/src/modules/pipeline/DynamicAOPass.ts index 8c030bb34..67f3e7c00 100644 --- a/packages/viewer/src/modules/pipeline/DynamicAOPass.ts +++ b/packages/viewer/src/modules/pipeline/DynamicAOPass.ts @@ -48,13 +48,13 @@ export interface DynamicAOPassParams { } export const DefaultDynamicAOPassParams = { - intensity: 1.25, + intensity: 1.5, scale: 0, - kernelRadius: 5, - bias: 0.15, + kernelRadius: 10, + bias: 0.2, normalsType: NormalsType.ACCURATE, blurEnabled: true, - blurRadius: 4, + blurRadius: 2, blurStdDev: 4, blurDepthCutoff: 0.0007 } diff --git a/packages/viewer/src/modules/pipeline/Pipeline.ts b/packages/viewer/src/modules/pipeline/Pipeline.ts index 260baa3b9..44ab59526 100644 --- a/packages/viewer/src/modules/pipeline/Pipeline.ts +++ b/packages/viewer/src/modules/pipeline/Pipeline.ts @@ -7,7 +7,7 @@ import Batcher from '../batching/Batcher' import SpeckleRenderer from '../SpeckleRenderer' import { ApplySAOPass } from './ApplyAOPass' import { CopyOutputPass } from './CopyOutputPass' -import { DepthPass, DepthType } from './DepthPass' +import { DepthPass, DepthSize, DepthType } from './DepthPass' import { NormalsPass } from './NormalsPass' import { DefaultDynamicAOPassParams, @@ -97,6 +97,7 @@ export class Pipeline { switch (outputType) { case PipelineOutputType.FINAL: pipeline = this.getDefaultPipeline() + this.depthPass.depthSize = DepthSize.FULL this.applySaoPass.setTexture('tDiffuse', this.staticAoPass.outputTexture) this.applySaoPass.setTexture('tDiffuseInterp', this.dynamicAoPass.outputTexture) this.needsProgressive = true @@ -105,6 +106,7 @@ export class Pipeline { case PipelineOutputType.DEPTH_RGBA: pipeline.push(this.depthPass) pipeline.push(this.copyOutputPass) + this.depthPass.depthSize = DepthSize.FULL this.copyOutputPass.setTexture('tDiffuse', this.depthPass.outputTexture) this.copyOutputPass.setOutputType(PipelineOutputType.DEPTH_RGBA) this.needsProgressive = false @@ -113,6 +115,7 @@ export class Pipeline { case PipelineOutputType.DEPTH: pipeline.push(this.depthPass) pipeline.push(this.copyOutputPass) + this.depthPass.depthSize = DepthSize.FULL this.copyOutputPass.setTexture('tDiffuse', this.depthPass.outputTexture) this.copyOutputPass.setOutputType(PipelineOutputType.DEPTH) this.needsProgressive = false @@ -137,6 +140,7 @@ export class Pipeline { pipeline.push(this.copyOutputPass) this.dynamicAoPass.enabled = true this.depthPass.depthType = DepthType.PERSPECTIVE_DEPTH + this.depthPass.depthSize = DepthSize.HALF this.dynamicAoPass.setOutputType(DynamicAOOutputType.RECONSTRUCTED_NORMALS) this.copyOutputPass.setTexture('tDiffuse', this.dynamicAoPass.outputTexture) this.copyOutputPass.setOutputType(PipelineOutputType.GEOMETRY_NORMALS) @@ -171,6 +175,7 @@ export class Pipeline { : false this.dynamicAoPass.enabled = true this.depthPass.depthType = DepthType.PERSPECTIVE_DEPTH + this.depthPass.depthSize = DepthSize.HALF this.copyOutputPass.setTexture('tDiffuse', this.dynamicAoPass.outputTexture) this.copyOutputPass.setOutputType(PipelineOutputType.COLOR) this.dynamicAoPass.setOutputType(DynamicAOOutputType.AO_BLURRED) @@ -184,6 +189,7 @@ export class Pipeline { pipeline.push(this.staticAoPass) pipeline.push(this.copyOutputPass) this.depthPass.depthType = DepthType.LINEAR_DEPTH + this.depthPass.depthSize = DepthSize.FULL this.copyOutputPass.setTexture('tDiffuse', this.staticAoPass.outputTexture) this.copyOutputPass.setOutputType(PipelineOutputType.COLOR) this.needsProgressive = true @@ -257,7 +263,7 @@ export class Pipeline { ) this.applySaoPass.renderToScreen = true - this.dynamicAoPass.setTexture('tDepth', this.depthPass.outputTexture) + this.dynamicAoPass.setTexture('tDepth', this.depthPass.outputTextureHalf) this.dynamicAoPass.setTexture('tNormal', this.normalsPass.outputTexture) this.applySaoPass.setTexture('tDiffuse', this.dynamicAoPass.outputTexture) this.applySaoPass.setTexture('tDiffuseInterp', this.dynamicAoPass.outputTexture) @@ -344,6 +350,7 @@ export class Pipeline { this.accumulationFrame = 0 this.depthPass.enabled = true this.depthPass.depthType = DepthType.LINEAR_DEPTH + this.depthPass.depthSize = DepthSize.FULL this.normalsPass.enabled = false this.dynamicAoPass.enabled = false this.renderPass.enabled = true @@ -361,6 +368,7 @@ export class Pipeline { this.accumulationFrame = 0 this.renderType = RenderType.NORMAL this.depthPass.depthType = DepthType.PERSPECTIVE_DEPTH + this.depthPass.depthSize = DepthSize.HALF this.staticAoPass.enabled = false this.applySaoPass.enabled = true this.dynamicAoPass.enabled = true From 98e5dbfe1568f589651214bf73ab45ba80e22e2d Mon Sep 17 00:00:00 2001 From: Alex Date: Thu, 27 Oct 2022 13:47:19 +0300 Subject: [PATCH 31/54] Changed some params now that dynamic AO is at half res --- packages/viewer-sandbox/src/Sandbox.ts | 8 ++++---- packages/viewer/src/modules/pipeline/DynamicAOPass.ts | 4 ++-- packages/viewer/src/modules/pipeline/StaticAOPass.ts | 4 ++-- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/packages/viewer-sandbox/src/Sandbox.ts b/packages/viewer-sandbox/src/Sandbox.ts index a9c66ffd9..cb0bf9dbc 100644 --- a/packages/viewer-sandbox/src/Sandbox.ts +++ b/packages/viewer-sandbox/src/Sandbox.ts @@ -37,18 +37,18 @@ export default class Sandbox { dynamicAoParams: { intensity: 1.5, scale: 0, - kernelRadius: 10, + kernelRadius: 5, bias: 0.2, normalsType: 2, blurEnabled: true, blurRadius: 2, blurStdDev: 4, - blurDepthCutoff: 0.0007 + blurDepthCutoff: 0.007 }, staticAoEnabled: true, staticAoParams: { - intensity: 0.8, - kernelRadius: 20, // Screen space + intensity: 1, + kernelRadius: 30, // Screen space kernelSize: 16, bias: 0.01, minDistance: 0, diff --git a/packages/viewer/src/modules/pipeline/DynamicAOPass.ts b/packages/viewer/src/modules/pipeline/DynamicAOPass.ts index 67f3e7c00..dda11ba67 100644 --- a/packages/viewer/src/modules/pipeline/DynamicAOPass.ts +++ b/packages/viewer/src/modules/pipeline/DynamicAOPass.ts @@ -50,13 +50,13 @@ export interface DynamicAOPassParams { export const DefaultDynamicAOPassParams = { intensity: 1.5, scale: 0, - kernelRadius: 10, + kernelRadius: 5, bias: 0.2, normalsType: NormalsType.ACCURATE, blurEnabled: true, blurRadius: 2, blurStdDev: 4, - blurDepthCutoff: 0.0007 + blurDepthCutoff: 0.007 } export class DynamicSAOPass extends Pass implements SpecklePass { diff --git a/packages/viewer/src/modules/pipeline/StaticAOPass.ts b/packages/viewer/src/modules/pipeline/StaticAOPass.ts index 35e3a3205..c84717a81 100644 --- a/packages/viewer/src/modules/pipeline/StaticAOPass.ts +++ b/packages/viewer/src/modules/pipeline/StaticAOPass.ts @@ -49,8 +49,8 @@ export interface StaticAoPassParams { } export const DefaultStaticAoPassParams = { - intensity: 0.8, - kernelRadius: 20, // Screen space + intensity: 1, + kernelRadius: 30, // Screen space kernelSize: 16, bias: 0.01, minDistance: 0, From 3b8a2a0c8c26a6c4dadb95fc2190b4ad7573d592 Mon Sep 17 00:00:00 2001 From: AlexandruPopovici Date: Thu, 27 Oct 2022 18:55:35 +0300 Subject: [PATCH 32/54] Made section boxes work properly with our new progressive pipeline --- packages/viewer-sandbox/src/main.ts | 6 ++-- packages/viewer/src/IViewer.ts | 5 ++-- packages/viewer/src/modules/DebugViewer.ts | 4 +-- packages/viewer/src/modules/SectionBox.js | 6 +++- .../viewer/src/modules/SpeckleRenderer.ts | 28 ++----------------- packages/viewer/src/modules/Viewer.ts | 21 ++++---------- .../src/modules/filtering/FilteringManager.ts | 2 +- .../viewer/src/modules/pipeline/DepthPass.ts | 6 +++- 8 files changed, 27 insertions(+), 51 deletions(-) diff --git a/packages/viewer-sandbox/src/main.ts b/packages/viewer-sandbox/src/main.ts index 3b48e5239..82d0cecc3 100644 --- a/packages/viewer-sandbox/src/main.ts +++ b/packages/viewer-sandbox/src/main.ts @@ -82,7 +82,7 @@ sandbox.makeFilteringUI() await sandbox.loadUrl( // 'https://speckle.xyz/streams/da9e320dad/commits/5388ef24b8?c=%5B-7.66134,10.82932,6.41935,-0.07739,-13.88552,1.8697,0,1%5D' // Revit sample house (good for bim-like stuff with many display meshes) - // 'https://speckle.xyz/streams/da9e320dad/commits/5388ef24b8' + 'https://speckle.xyz/streams/da9e320dad/commits/5388ef24b8' // 'Super' heavy revit shit // 'https://speckle.xyz/streams/e6f9156405/commits/0694d53bb5' // IFC building (good for a tree based structure) @@ -97,7 +97,7 @@ await sandbox.loadUrl( // AutoCAD // 'https://latest.speckle.dev/streams/3ed8357f29/commits/d10f2af1ce' //Blizzard world - 'https://latest.speckle.dev/streams/0c6ad366c4/commits/aa1c393aec' + // 'https://latest.speckle.dev/streams/0c6ad366c4/commits/aa1c393aec' //Car // 'https://latest.speckle.dev/streams/17d2e25a97/commits/6b6cf3d43e' // Jonathon's @@ -120,4 +120,6 @@ await sandbox.loadUrl( //'https://latest.speckle.dev/streams/0c6ad366c4/commits/912d83412e' // Freezers // 'https://speckle.xyz/streams/f0532359ac/commits/98678e2a3d?c=%5B2455.15367,2689.87156,4366.68444,205.422,-149.41199,148.749,0,1%5D' + //Gergo's house + // 'https://latest.speckle.dev/streams/c1faab5c62/commits/78bdd8eb76' ) diff --git a/packages/viewer/src/IViewer.ts b/packages/viewer/src/IViewer.ts index 07ca0b93e..37ee844ec 100644 --- a/packages/viewer/src/IViewer.ts +++ b/packages/viewer/src/IViewer.ts @@ -46,7 +46,8 @@ export enum ViewerEvent { UnloadComplete = 'unload-complete', UnloadAllComplete = 'unload-all-complete', Busy = 'busy', - SectionBoxChanged = 'section-box-changed' + SectionBoxChanged = 'section-box-changed', + SectionBoxUpdated = 'section-box-updated' } export type SelectionEvent = { @@ -117,7 +118,7 @@ export interface IViewer { init(): Promise resize(): void on(eventType: ViewerEvent, handler: (arg) => void) - + requestRender(): void setSectionBox( box?: { min: { x: number; y: number; z: number } diff --git a/packages/viewer/src/modules/DebugViewer.ts b/packages/viewer/src/modules/DebugViewer.ts index eff6b8055..e3a66c231 100644 --- a/packages/viewer/src/modules/DebugViewer.ts +++ b/packages/viewer/src/modules/DebugViewer.ts @@ -4,9 +4,7 @@ export class DebugViewer extends Viewer { getRenderer() { return this.speckleRenderer } - requestRender() { - this.speckleRenderer.needsRender = true - } + requestRenderShadowmap() { this.getRenderer().updateDirectLights() } diff --git a/packages/viewer/src/modules/SectionBox.js b/packages/viewer/src/modules/SectionBox.js index f77eef0c9..0ca744058 100644 --- a/packages/viewer/src/modules/SectionBox.js +++ b/packages/viewer/src/modules/SectionBox.js @@ -153,10 +153,11 @@ export default class SectionBox { _draggingChangeHandler() { if (!this.display.visible) return this.boxHelper.update() - this._generateOrUpdatePlanes() + // this._generateOrUpdatePlanes() // Dragging a side / plane if (this.dragging && this.currentRange) { + this._generateOrUpdatePlanes() if (this.prevPosition === null) this.prevPosition = this.hoverPlane.position.clone() this.prevPosition.sub(this.hoverPlane.position) @@ -178,6 +179,7 @@ export default class SectionBox { // Dragging the whole section box if (this.dragging && !this.currentRange) { + this._generateOrUpdatePlanes() if (this.prevPosition === null) this.prevPosition = this.sphere.position.clone() this.prevPosition.sub(this.sphere.position) this.prevPosition.negate() @@ -196,6 +198,7 @@ export default class SectionBox { } this.viewer.needsRender = true this.viewer.emit('section-box-changed', this.getCurrentBox()) + this.viewer.requestRender() } _clickHandler(args) { @@ -327,6 +330,7 @@ export default class SectionBox { plane.setFromCoplanarPoints(a, b, c) index++ } + this.viewer.emit('section-box-updated', this.getCurrentBox()) } _attachControlsToBox() { diff --git a/packages/viewer/src/modules/SpeckleRenderer.ts b/packages/viewer/src/modules/SpeckleRenderer.ts index 2a058dcab..72a4a16a2 100644 --- a/packages/viewer/src/modules/SpeckleRenderer.ts +++ b/packages/viewer/src/modules/SpeckleRenderer.ts @@ -312,31 +312,6 @@ export default class SpeckleRenderer { this.viewer.cameraHandler.activeCam.camera.far = d this.viewer.cameraHandler.activeCam.camera.updateProjectionMatrix() this.viewer.cameraHandler.camera.updateProjectionMatrix() - // const currentAzimuth = this.viewer.cameraHandler.controls.azimuthAngle - // const currentPolar = this.viewer.cameraHandler.controls.polarAngle - // const currentPosition = this.viewer.cameraHandler.activeCam.camera.position - // const dAzimuth = Math.max(0, Math.abs(currentAzimuth - this.lastAzimuth)) - // const dPolar = Math.max(0, Math.abs(currentPolar - this.lastPolar)) - // const dPosition = Math.max(0, currentPosition.distanceTo(this.lastCameraPosition)) - // const motionMaxDelta = Math.max(0, Math.max(dAzimuth, dPolar, dPosition)) - // // console.log(dPosition, this.lastCameraMotionDelta) - // console.log( - // currentPosition.distanceTo(this.viewer.cameraHandler.controls['_targetEnd']) - // ) - // if (motionMaxDelta === 0 && this.lastCameraMotionDelta > 0) { - // this.pipeline.onStationaryBegin() - // } - // if ( - // motionMaxDelta > this.POSITION_RESUME_EPSILON && - // this.lastCameraMotionDelta === 0 - // ) { - // this._needsRender = true - // this.pipeline.onStationaryEnd() - // } - // this.lastCameraMotionDelta = motionMaxDelta - // this.lastAzimuth = currentAzimuth - // this.lastPolar = currentPolar - // this.lastCameraPosition.copy(currentPosition) this.pipeline.update(this) } @@ -455,6 +430,7 @@ export default class SpeckleRenderer { }) this.pipeline.updateClippingPlanes(planes) this.renderer.shadowMap.needsUpdate = true + this.resetPipeline() } private addDirectLights() { @@ -544,7 +520,7 @@ export default class SpeckleRenderer { this.sun.shadow.camera.far = Math.abs(lightSpaceBox.min.z) this.sun.shadow.camera.updateProjectionMatrix() this.renderer.shadowMap.needsUpdate = true - this.viewer.needsRender = true + this.needsRender = true this.updateHelpers() } diff --git a/packages/viewer/src/modules/Viewer.ts b/packages/viewer/src/modules/Viewer.ts index 93829b213..47b4062a0 100644 --- a/packages/viewer/src/modules/Viewer.ts +++ b/packages/viewer/src/modules/Viewer.ts @@ -45,22 +45,11 @@ export class Viewer extends EventEmitter implements IViewer { public sectionBox: SectionBox public cameraHandler: CameraHandler - /** Render flag for on-demand rendering */ - private _needsRender: boolean - /** Misc members */ private inProgressOperations: number private clock: Clock private loaders: { [id: string]: ViewerObjectLoader } = {} - public get needsRender(): boolean { - return this._needsRender - } - - public set needsRender(value: boolean) { - this._needsRender = value || this._needsRender - } - /** Gets the World object. Currently it's used for info mostly */ public static get World(): World { return this.world @@ -96,13 +85,12 @@ export class Viewer extends EventEmitter implements IViewer { this.sectionBox = new SectionBox(this) this.sectionBox.off() - this.sectionBox.controls.addEventListener('change', () => { + this.on(ViewerEvent.SectionBoxUpdated, () => { this.speckleRenderer.updateClippingPlanes(this.sectionBox.planes) }) this.frame() this.resize() - this.needsRender = true this.on(ViewerEvent.LoadComplete, (url) => { WorldTree.getRenderTree(url).buildRenderTree() @@ -138,7 +126,11 @@ export class Viewer extends EventEmitter implements IViewer { const width = this.container.offsetWidth const height = this.container.offsetHeight this.speckleRenderer.resize(width, height) - this.needsRender = true + } + + public requestRender() { + this.speckleRenderer.needsRender = true + this.speckleRenderer.resetPipeline() } private frame() { @@ -343,7 +335,6 @@ export class Viewer extends EventEmitter implements IViewer { transition = true ): void { this.speckleRenderer.setView(view, transition) - this.needsRender = true } public screenshot(): Promise { diff --git a/packages/viewer/src/modules/filtering/FilteringManager.ts b/packages/viewer/src/modules/filtering/FilteringManager.ts index 1e85753b9..a0e927b64 100644 --- a/packages/viewer/src/modules/filtering/FilteringManager.ts +++ b/packages/viewer/src/modules/filtering/FilteringManager.ts @@ -451,7 +451,7 @@ export class FilteringManager { } this.Renderer.endFilter() - this.Renderer.viewer.needsRender = true + this.Renderer.viewer.requestRender() return returnState } diff --git a/packages/viewer/src/modules/pipeline/DepthPass.ts b/packages/viewer/src/modules/pipeline/DepthPass.ts index 7bc94a8ed..e0b4f3b5a 100644 --- a/packages/viewer/src/modules/pipeline/DepthPass.ts +++ b/packages/viewer/src/modules/pipeline/DepthPass.ts @@ -118,7 +118,11 @@ export class DepthPass extends Pass implements SpecklePass { const originalClearAlpha = renderer.getClearAlpha() const originalAutoClear = renderer.autoClear - renderer.setRenderTarget(this.depthBufferSize === DepthSize.FULL ? this.renderTarget: this.renderTargetHalf) + renderer.setRenderTarget( + this.depthBufferSize === DepthSize.FULL + ? this.renderTarget + : this.renderTargetHalf + ) renderer.autoClear = false renderer.setClearColor(0x000000) From a140153a583453b7a2b8d9b663f14337143e1db9 Mon Sep 17 00:00:00 2001 From: AlexandruPopovici Date: Fri, 28 Oct 2022 12:31:19 +0300 Subject: [PATCH 33/54] Added correct kernel size computation for orthographic projection --- .../speckle-static-ao-generate-frag.ts | 26 ++++++++++++------- .../viewer/src/modules/pipeline/Pipeline.ts | 1 + 2 files changed, 18 insertions(+), 9 deletions(-) diff --git a/packages/viewer/src/modules/materials/shaders/speckle-static-ao-generate-frag.ts b/packages/viewer/src/modules/materials/shaders/speckle-static-ao-generate-frag.ts index cf3d48dc4..83dba5c99 100644 --- a/packages/viewer/src/modules/materials/shaders/speckle-static-ao-generate-frag.ts +++ b/packages/viewer/src/modules/materials/shaders/speckle-static-ao-generate-frag.ts @@ -47,9 +47,13 @@ export const speckleStaticAoGenerateFrag = /* glsl */ ` float getPerspectiveDepth(const in vec2 coords) { float linearDepth = unpackRGBAToDepth( texture2D( tDepth, coords ) ); - float viewZ = orthographicDepthToViewZ(linearDepth, cameraNear, cameraFar); - float centerDepth = viewZToPerspectiveDepth(viewZ, cameraNear, cameraFar); - return centerDepth; + #if PERSPECTIVE_CAMERA == 1 + float viewZ = orthographicDepthToViewZ(linearDepth, cameraNear, cameraFar); + float centerDepth = viewZToPerspectiveDepth(viewZ, cameraNear, cameraFar); + return centerDepth; + #else + return linearDepth; + #endif } float getViewDepth(const in float linearDepth) { @@ -194,12 +198,16 @@ export const speckleStaticAoGenerateFrag = /* glsl */ ` const float offset = PI2 / float(NUM_FRAMES); float computeKernelSize(float d, float r) { - // Apparently this is wrong - // return (r * tan(fov) * d) / (size.y * 0.5); - // And this is correct - float rp = r / (size.y * 0.5); - return sqrt((rp*rp*tanFov*tanFov*d*d)/(1. + rp*rp*tanFov*tanFov)); - // return r; + #if PERSPECTIVE_CAMERA == 1 + // Apparently this is wrong + // return (r * tan(fov) * d) / (size.y * 0.5); + // And this is correct + float rp = r / (size.y * 0.5); + return sqrt((rp*rp*tanFov*tanFov*d*d)/(1. + rp*rp*tanFov*tanFov)); + #else + float twoOrthoSize = size.y / (2./ cameraProjectionMatrix[1][1]); + return r / twoOrthoSize; + #endif } float getAmbientOcclusion( const in vec3 centerViewPosition, in float centerDepth ) { diff --git a/packages/viewer/src/modules/pipeline/Pipeline.ts b/packages/viewer/src/modules/pipeline/Pipeline.ts index 44ab59526..3ed0062f5 100644 --- a/packages/viewer/src/modules/pipeline/Pipeline.ts +++ b/packages/viewer/src/modules/pipeline/Pipeline.ts @@ -304,6 +304,7 @@ export class Pipeline { } public update(renderer: SpeckleRenderer) { + this.renderPass.camera = renderer.camera this.depthPass.update(renderer.scene, renderer.camera) this.dynamicAoPass.update(renderer.scene, renderer.camera) this.normalsPass.update(renderer.scene, renderer.camera) From 2e4c65990f68b04688ef3ebbf3ba96165ad1c615 Mon Sep 17 00:00:00 2001 From: AlexandruPopovici Date: Fri, 28 Oct 2022 12:37:02 +0300 Subject: [PATCH 34/54] Patched the camera controls to send events when zooming in orthographic mode --- .../src/modules/objects/SpeckleCameraControls.ts | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/packages/viewer/src/modules/objects/SpeckleCameraControls.ts b/packages/viewer/src/modules/objects/SpeckleCameraControls.ts index 82823ea02..1b59bb151 100644 --- a/packages/viewer/src/modules/objects/SpeckleCameraControls.ts +++ b/packages/viewer/src/modules/objects/SpeckleCameraControls.ts @@ -96,6 +96,22 @@ export class SpeckleCameraControls extends CameraControls { return } + protected _zoomInternal = (delta: number, x: number, y: number): void => { + const zoomScale = Math.pow(0.95, delta * this.dollySpeed) + + // for both PerspectiveCamera and OrthographicCamera + this.zoomTo(this._zoom * zoomScale) + this._didDolly = true + this.dispatchEvent({ type: 'controlstart' }) + if (this.dollyToCursor) { + this._dollyControlAmount = this._zoomEnd + + this._dollyControlCoord.set(x, y) + } + + return + } + /** * Dolly in/out camera position to given distance. * @param distance Distance of dolly. From b3ad159cd69013ddcabe00b2484c4b51f1cdbc87 Mon Sep 17 00:00:00 2001 From: AlexandruPopovici Date: Fri, 28 Oct 2022 14:03:18 +0300 Subject: [PATCH 35/54] Some more integrations with camera views and fixed a bug with bad pipeline resets --- packages/viewer/src/modules/SpeckleRenderer.ts | 7 ++++--- packages/viewer/src/modules/Viewer.ts | 1 + 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/packages/viewer/src/modules/SpeckleRenderer.ts b/packages/viewer/src/modules/SpeckleRenderer.ts index 72a4a16a2..afbc27555 100644 --- a/packages/viewer/src/modules/SpeckleRenderer.ts +++ b/packages/viewer/src/modules/SpeckleRenderer.ts @@ -316,9 +316,9 @@ export default class SpeckleRenderer { this.pipeline.update(this) } - public resetPipeline() { + public resetPipeline(force = false) { this._needsRender = true - this.pipeline.reset() + if (this.viewer.cameraHandler.controls.hasRested || force) this.pipeline.reset() } public render(): void { @@ -384,7 +384,7 @@ export default class SpeckleRenderer { this.updateDirectLights() this.updateHelpers() - this.resetPipeline() + this.resetPipeline(true) } public removeRenderTree(subtreeId: string) { @@ -800,6 +800,7 @@ export default class SpeckleRenderer { if (this.isPolarView(view)) { this.setViewPolar(view, transition) } + this.pipeline.onStationaryEnd() } private setViewSpeckle(view: SpeckleView, transition = true) { diff --git a/packages/viewer/src/modules/Viewer.ts b/packages/viewer/src/modules/Viewer.ts index 47b4062a0..3fbf27fd3 100644 --- a/packages/viewer/src/modules/Viewer.ts +++ b/packages/viewer/src/modules/Viewer.ts @@ -310,6 +310,7 @@ export class Viewer extends EventEmitter implements IViewer { public toggleCameraProjection() { this.cameraHandler.toggleCameras() + this.speckleRenderer.resetPipeline() } public setLightConfiguration(config: SunLightConfiguration): void { From 22e494d4d276bfc7bf6c7e15eae802a24ae0bc94 Mon Sep 17 00:00:00 2001 From: Alex Date: Fri, 28 Oct 2022 14:54:36 +0300 Subject: [PATCH 36/54] Small fix for zoom and startup reset --- packages/viewer/src/modules/SpeckleRenderer.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/viewer/src/modules/SpeckleRenderer.ts b/packages/viewer/src/modules/SpeckleRenderer.ts index afbc27555..e193bb891 100644 --- a/packages/viewer/src/modules/SpeckleRenderer.ts +++ b/packages/viewer/src/modules/SpeckleRenderer.ts @@ -384,7 +384,8 @@ export default class SpeckleRenderer { this.updateDirectLights() this.updateHelpers() - this.resetPipeline(true) + // this.resetPipeline(true) + this._needsRender = true } public removeRenderTree(subtreeId: string) { @@ -682,9 +683,11 @@ export default class SpeckleRenderer { public zoom(objectIds?: string[], fit?: number, transition?: boolean) { if (!objectIds) { this.zoomExtents(fit, transition) + this.pipeline.onStationaryEnd() return } this.zoomToBox(this.boxFromObjects(objectIds), fit, transition) + this.pipeline.onStationaryEnd() } /** Taken from InteractionsHandler. Will revisit in the future */ From a3ddd4fb236a271ca625b634dd834acc97c90a08 Mon Sep 17 00:00:00 2001 From: AlexandruPopovici Date: Fri, 28 Oct 2022 15:35:12 +0300 Subject: [PATCH 37/54] Various fixes of issues I could find --- packages/viewer-sandbox/src/main.ts | 4 +-- .../viewer/src/modules/SpeckleRenderer.ts | 18 ++++++++++++- .../viewer/src/modules/pipeline/Pipeline.ts | 26 +++++++++++-------- 3 files changed, 34 insertions(+), 14 deletions(-) diff --git a/packages/viewer-sandbox/src/main.ts b/packages/viewer-sandbox/src/main.ts index 82d0cecc3..712745eff 100644 --- a/packages/viewer-sandbox/src/main.ts +++ b/packages/viewer-sandbox/src/main.ts @@ -82,7 +82,7 @@ sandbox.makeFilteringUI() await sandbox.loadUrl( // 'https://speckle.xyz/streams/da9e320dad/commits/5388ef24b8?c=%5B-7.66134,10.82932,6.41935,-0.07739,-13.88552,1.8697,0,1%5D' // Revit sample house (good for bim-like stuff with many display meshes) - 'https://speckle.xyz/streams/da9e320dad/commits/5388ef24b8' + // 'https://speckle.xyz/streams/da9e320dad/commits/5388ef24b8' // 'Super' heavy revit shit // 'https://speckle.xyz/streams/e6f9156405/commits/0694d53bb5' // IFC building (good for a tree based structure) @@ -117,7 +117,7 @@ await sandbox.loadUrl( // REVIT test stream // 'https://latest.speckle.dev/streams/c544db35f5/commits/7c29374369' // Arcs - //'https://latest.speckle.dev/streams/0c6ad366c4/commits/912d83412e' + 'https://latest.speckle.dev/streams/0c6ad366c4/commits/912d83412e' // Freezers // 'https://speckle.xyz/streams/f0532359ac/commits/98678e2a3d?c=%5B2455.15367,2689.87156,4366.68444,205.422,-149.41199,148.749,0,1%5D' //Gergo's house diff --git a/packages/viewer/src/modules/SpeckleRenderer.ts b/packages/viewer/src/modules/SpeckleRenderer.ts index e193bb891..cc74c2d2b 100644 --- a/packages/viewer/src/modules/SpeckleRenderer.ts +++ b/packages/viewer/src/modules/SpeckleRenderer.ts @@ -44,7 +44,12 @@ import { SunLightConfiguration, ViewerEvent } from '../IViewer' -import { DefaultPipelineOptions, Pipeline, PipelineOptions } from './pipeline/Pipeline' +import { + DefaultPipelineOptions, + Pipeline, + PipelineOptions, + RenderType +} from './pipeline/Pipeline' export default class SpeckleRenderer { private readonly SHOW_HELPERS = false @@ -210,6 +215,15 @@ export default class SpeckleRenderer { this._needsRender = true this.pipeline.onStationaryEnd() }) + this.viewer.cameraHandler.controls.addEventListener('update', () => { + if ( + !this.viewer.cameraHandler.controls.hasRested && + this.pipeline.renderType === RenderType.ACCUMULATION + ) { + this._needsRender = true + this.pipeline.onStationaryEnd() + } + }) } public update(deltaTime: number) { @@ -412,6 +426,7 @@ export default class SpeckleRenderer { public endFilter() { this.batcher.autoFillDrawRanges(this.filterBatchRecording) + this.updateClippingPlanes(this.viewer.sectionBox.planes) this.renderer.shadowMap.needsUpdate = true } @@ -523,6 +538,7 @@ export default class SpeckleRenderer { this.renderer.shadowMap.needsUpdate = true this.needsRender = true this.updateHelpers() + this.resetPipeline() } public setSunLightConfiguration(config: SunLightConfiguration) { diff --git a/packages/viewer/src/modules/pipeline/Pipeline.ts b/packages/viewer/src/modules/pipeline/Pipeline.ts index 3ed0062f5..3cc3d73a9 100644 --- a/packages/viewer/src/modules/pipeline/Pipeline.ts +++ b/packages/viewer/src/modules/pipeline/Pipeline.ts @@ -78,7 +78,7 @@ export class Pipeline { private staticAoPass: StaticAOPass = null private drawingSize: Vector2 = new Vector2() - private renderType: RenderType = RenderType.NORMAL + private _renderType: RenderType = RenderType.NORMAL private accumulationFrame = 0 public set pipelineOptions(options: Partial) { @@ -202,12 +202,16 @@ export class Pipeline { public set needsProgressive(value: boolean) { this._needsProgressive = value - if (!value) this.renderType = RenderType.NORMAL - if (value && this.renderType === RenderType.NORMAL) - this.renderType = RenderType.ACCUMULATION + if (!value) this._renderType = RenderType.NORMAL + if (value && this._renderType === RenderType.NORMAL) + this._renderType = RenderType.ACCUMULATION this.accumulationFrame = 0 } + public get renderType() { + return this._renderType + } + public constructor(renderer: WebGLRenderer, batcher: Batcher) { this._renderer = renderer this._batcher = batcher @@ -320,7 +324,7 @@ export class Pipeline { if (this.drawingSize.length() === 0) return this._renderer.clear(true) - if (this.renderType === RenderType.NORMAL) { + if (this._renderType === RenderType.NORMAL) { this.composer.render() const ret = false || this._resetFrame if (this._resetFrame) { @@ -343,11 +347,11 @@ export class Pipeline { public onStationaryBegin() { if (!this._needsProgressive) return - if (this.renderType === RenderType.ACCUMULATION) { + if (this._renderType === RenderType.ACCUMULATION) { this.accumulationFrame = 0 return } - this.renderType = RenderType.ACCUMULATION + this._renderType = RenderType.ACCUMULATION this.accumulationFrame = 0 this.depthPass.enabled = true this.depthPass.depthType = DepthType.LINEAR_DEPTH @@ -359,22 +363,22 @@ export class Pipeline { this.staticAoPass.enabled = true this.applySaoPass.setTexture('tDiffuse', this.staticAoPass.outputTexture) this.applySaoPass.setTexture('tDiffuseInterp', this.dynamicAoPass.outputTexture) - this.applySaoPass.setRenderType(this.renderType) + this.applySaoPass.setRenderType(this._renderType) console.warn('Starting stationary') } public onStationaryEnd() { if (!this._needsProgressive) return - if (this.renderType === RenderType.NORMAL) return + if (this._renderType === RenderType.NORMAL) return this.accumulationFrame = 0 - this.renderType = RenderType.NORMAL + this._renderType = RenderType.NORMAL this.depthPass.depthType = DepthType.PERSPECTIVE_DEPTH this.depthPass.depthSize = DepthSize.HALF this.staticAoPass.enabled = false this.applySaoPass.enabled = true this.dynamicAoPass.enabled = true this.applySaoPass.setTexture('tDiffuse', this.dynamicAoPass.outputTexture) - this.applySaoPass.setRenderType(this.renderType) + this.applySaoPass.setRenderType(this._renderType) console.warn('Ending stationary') } } From ee6df9484cc58f8e15429f8801b5a173d0c02ae8 Mon Sep 17 00:00:00 2001 From: AlexandruPopovici Date: Fri, 28 Oct 2022 15:41:22 +0300 Subject: [PATCH 38/54] Removed longs --- packages/viewer-sandbox/src/main.ts | 4 ++-- packages/viewer/src/modules/pipeline/Pipeline.ts | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/viewer-sandbox/src/main.ts b/packages/viewer-sandbox/src/main.ts index 712745eff..25416f73c 100644 --- a/packages/viewer-sandbox/src/main.ts +++ b/packages/viewer-sandbox/src/main.ts @@ -82,7 +82,7 @@ sandbox.makeFilteringUI() await sandbox.loadUrl( // 'https://speckle.xyz/streams/da9e320dad/commits/5388ef24b8?c=%5B-7.66134,10.82932,6.41935,-0.07739,-13.88552,1.8697,0,1%5D' // Revit sample house (good for bim-like stuff with many display meshes) - // 'https://speckle.xyz/streams/da9e320dad/commits/5388ef24b8' + 'https://speckle.xyz/streams/da9e320dad/commits/5388ef24b8' // 'Super' heavy revit shit // 'https://speckle.xyz/streams/e6f9156405/commits/0694d53bb5' // IFC building (good for a tree based structure) @@ -117,7 +117,7 @@ await sandbox.loadUrl( // REVIT test stream // 'https://latest.speckle.dev/streams/c544db35f5/commits/7c29374369' // Arcs - 'https://latest.speckle.dev/streams/0c6ad366c4/commits/912d83412e' + // 'https://latest.speckle.dev/streams/0c6ad366c4/commits/912d83412e' // Freezers // 'https://speckle.xyz/streams/f0532359ac/commits/98678e2a3d?c=%5B2455.15367,2689.87156,4366.68444,205.422,-149.41199,148.749,0,1%5D' //Gergo's house diff --git a/packages/viewer/src/modules/pipeline/Pipeline.ts b/packages/viewer/src/modules/pipeline/Pipeline.ts index 3cc3d73a9..f0d119092 100644 --- a/packages/viewer/src/modules/pipeline/Pipeline.ts +++ b/packages/viewer/src/modules/pipeline/Pipeline.ts @@ -333,7 +333,7 @@ export class Pipeline { } return ret } else { - console.warn('Rendering accumulation frame -> ', this.accumulationFrame) + // console.warn('Rendering accumulation frame -> ', this.accumulationFrame) this.composer.render() this.accumulationFrame++ return this.accumulationFrame < Pipeline.ACCUMULATE_FRAMES @@ -364,7 +364,7 @@ export class Pipeline { this.applySaoPass.setTexture('tDiffuse', this.staticAoPass.outputTexture) this.applySaoPass.setTexture('tDiffuseInterp', this.dynamicAoPass.outputTexture) this.applySaoPass.setRenderType(this._renderType) - console.warn('Starting stationary') + // console.warn('Starting stationary') } public onStationaryEnd() { @@ -379,6 +379,6 @@ export class Pipeline { this.dynamicAoPass.enabled = true this.applySaoPass.setTexture('tDiffuse', this.dynamicAoPass.outputTexture) this.applySaoPass.setRenderType(this._renderType) - console.warn('Ending stationary') + // console.warn('Ending stationary') } } From 6d85acb9bb2a4869bf7a0e9ba5dd21a766b1faa6 Mon Sep 17 00:00:00 2001 From: AlexandruPopovici Date: Fri, 28 Oct 2022 17:54:27 +0300 Subject: [PATCH 39/54] preview-service now waits for the pipeline to convgerge before taking the screenshot --- packages/preview-service/render_page/src/app.js | 4 ++-- packages/preview-service/routes/preview.js | 5 +++-- packages/viewer-sandbox/src/Sandbox.ts | 1 + 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/packages/preview-service/render_page/src/app.js b/packages/preview-service/render_page/src/app.js index 554b567e7..ad52ee7d5 100644 --- a/packages/preview-service/render_page/src/app.js +++ b/packages/preview-service/render_page/src/app.js @@ -1,6 +1,6 @@ -import { Viewer, DefaultViewerParams } from '@speckle/viewer' +import { DebugViewer, DefaultViewerParams } from '@speckle/viewer' -const v = new Viewer(document.getElementById('renderer'), DefaultViewerParams) +const v = new DebugViewer(document.getElementById('renderer'), DefaultViewerParams) window.v = v // v.on( ViewerEvent.LoadProgress, args => console.log( args ) ) diff --git a/packages/preview-service/routes/preview.js b/packages/preview-service/routes/preview.js index 75384a6e7..de14e74c7 100644 --- a/packages/preview-service/routes/preview.js +++ b/packages/preview-service/routes/preview.js @@ -29,10 +29,11 @@ async function pageFunction(objectUrl) { window.v.zoom(undefined, 0.95, false) await waitForAnimation(100) - // full 360 + // full 360. This needs testing with the new progressive pipeline for (let i = 0; i < 24; i++) { window.v.setView({ azimuth: Math.PI / 12, polar: 0 }, false) - await waitForAnimation() + window.v.getRenderer().resetPipeline(true) + await waitForAnimation(1000) ret.scr[i + ''] = await window.v.screenshot() } diff --git a/packages/viewer-sandbox/src/Sandbox.ts b/packages/viewer-sandbox/src/Sandbox.ts index cb0bf9dbc..85d769347 100644 --- a/packages/viewer-sandbox/src/Sandbox.ts +++ b/packages/viewer-sandbox/src/Sandbox.ts @@ -265,6 +265,7 @@ export default class Sandbox { }) for (let i = 0; i < 24; i++) { this.viewer.setView({ azimuth: Math.PI / 12, polar: 0 }, false) + this.viewer.getRenderer().resetPipeline(true) await waitForAnimation(1000) } }) From 62b10797affc841224cdc6b22e86ad854ceea29a Mon Sep 17 00:00:00 2001 From: AlexandruPopovici Date: Mon, 31 Oct 2022 14:41:36 +0200 Subject: [PATCH 40/54] Handled #1110. Added the concept of 'ObjectLayers' which uses three's existing implementation for selective layer rendering/lighting/picking. Also added the BaseSpecklePass abstract class which provides a default implementation for any subclassing pass that needs to use specific layers when rendering. --- packages/viewer-sandbox/src/main.ts | 2 + packages/viewer/src/modules/SectionBox.js | 12 ++++ .../viewer/src/modules/SpeckleRenderer.ts | 14 +++- .../src/modules/legacy/SelectionHelper.js | 2 + .../src/modules/objects/SpeckleRaycaster.ts | 2 + .../viewer/src/modules/pipeline/ColorPass.ts | 71 +++++++++++++++++-- .../viewer/src/modules/pipeline/DepthPass.ts | 6 +- .../src/modules/pipeline/NormalsPass.ts | 6 +- .../viewer/src/modules/pipeline/Pipeline.ts | 14 ++-- .../src/modules/pipeline/SpecklePass.ts | 33 +++++++++ 10 files changed, 144 insertions(+), 18 deletions(-) diff --git a/packages/viewer-sandbox/src/main.ts b/packages/viewer-sandbox/src/main.ts index af389e925..349a1cb3a 100644 --- a/packages/viewer-sandbox/src/main.ts +++ b/packages/viewer-sandbox/src/main.ts @@ -124,4 +124,6 @@ await sandbox.loadUrl( // 'https://latest.speckle.dev/streams/c1faab5c62/commits/78bdd8eb76' // Point cloud // 'https://latest.speckle.dev/streams/2d19273d31/commits/9ceb423feb' + // Luis sphere + // 'https://speckle.xyz/streams/b85d53c3b4/commits/b47f21b707' ) diff --git a/packages/viewer/src/modules/SectionBox.js b/packages/viewer/src/modules/SectionBox.js index 0ca744058..e785005c3 100644 --- a/packages/viewer/src/modules/SectionBox.js +++ b/packages/viewer/src/modules/SectionBox.js @@ -3,6 +3,7 @@ import SelectionHelper from './legacy/SelectionHelper' import { TransformControls } from 'three/examples/jsm/controls/TransformControls.js' import { Box3 } from 'three' import { ViewerEvent } from '../IViewer' +import { ObjectLayers } from './SpeckleRenderer' export default class SectionBox { constructor(viewer) { @@ -13,6 +14,7 @@ export default class SectionBox { this.dragging = false this.display = new THREE.Group() this.display.name = 'SectionBox' + this.display.layers.set(ObjectLayers.PROPS) this.viewer.speckleRenderer.scene.add(this.display) // box @@ -25,11 +27,13 @@ export default class SectionBox { }) this.cube = new THREE.Mesh(this.boxGeometry, this.material) this.cube.visible = false + this.cube.layers.set(ObjectLayers.PROPS) this.display.add(this.cube) this.boxHelper = new THREE.BoxHelper(this.cube, 0x0a66ff) this.boxHelper.material.opacity = 0.4 + this.boxHelper.layers.set(ObjectLayers.PROPS) this.display.add(this.boxHelper) // we're attaching the gizmo mover to this sphere in the box centre @@ -38,6 +42,7 @@ export default class SectionBox { sphere, new THREE.MeshStandardMaterial({ color: 0x00ffff }) ) + this.sphere.layers.set(ObjectLayers.PROPS) this.sphere.visible = false this.display.add(this.sphere) @@ -56,6 +61,7 @@ export default class SectionBox { }) ) this.hoverPlane.visible = false + this.hoverPlane.layers.set(ObjectLayers.PROPS) this.display.add(this.hoverPlane) this.dragging = false @@ -125,6 +131,12 @@ export default class SectionBox { this.viewer.cameraHandler.activeCam.camera, this.viewer.speckleRenderer.renderer.domElement ) + for (let k = 0; k < this.controls.children.length; k++) { + this.controls.children[k].traverse((obj) => { + obj.layers.set(ObjectLayers.PROPS) + }) + } + this.controls.getRaycaster().layers.set(ObjectLayers.PROPS) this.controls.setSize(0.75) this.display.add(this.controls) this.controls.addEventListener('change', this._draggingChangeHandler.bind(this)) diff --git a/packages/viewer/src/modules/SpeckleRenderer.ts b/packages/viewer/src/modules/SpeckleRenderer.ts index cc74c2d2b..e29a503af 100644 --- a/packages/viewer/src/modules/SpeckleRenderer.ts +++ b/packages/viewer/src/modules/SpeckleRenderer.ts @@ -51,6 +51,11 @@ import { RenderType } from './pipeline/Pipeline' +export enum ObjectLayers { + STREAM_CONTENT = 1, + PROPS = 2 +} + export default class SpeckleRenderer { private readonly SHOW_HELPERS = false private readonly ANGLE_EPSILON = 0.0001 @@ -140,6 +145,7 @@ export default class SpeckleRenderer { this._scene = new Scene() this.rootGroup = new Group() this.rootGroup.name = 'ContentGroup' + this.rootGroup.layers.set(ObjectLayers.STREAM_CONTENT) this._scene.add(this.rootGroup) this.batcher = new Batcher() @@ -168,7 +174,7 @@ export default class SpeckleRenderer { container.appendChild(this._renderer.domElement) this.pipeline = new Pipeline(this._renderer, this.batcher) - this.pipeline.configure(this._scene, this.viewer.cameraHandler.activeCam.camera) + this.pipeline.configure() this.pipeline.pipelineOptions = DefaultPipelineOptions this.input = new Input(this._renderer.domElement, InputOptionsDefault) @@ -184,14 +190,17 @@ export default class SpeckleRenderer { const sceneBoxHelper = new Box3Helper(this.sceneBox, new Color(0x0000ff)) sceneBoxHelper.name = 'SceneBoxHelper' + sceneBoxHelper.layers.set(ObjectLayers.PROPS) helpers.add(sceneBoxHelper) const dirLightHelper = new DirectionalLightHelper(this.sun, 50, 0xff0000) dirLightHelper.name = 'DirLightHelper' + dirLightHelper.layers.set(ObjectLayers.PROPS) helpers.add(dirLightHelper) const camHelper = new CameraHelper(this.sun.shadow.camera) camHelper.name = 'CamHelper' + camHelper.layers.set(ObjectLayers.PROPS) helpers.add(camHelper) } @@ -376,11 +385,13 @@ export default class SpeckleRenderer { const subtreeGroup = new Group() subtreeGroup.name = subtreeId + subtreeGroup.layers.set(ObjectLayers.STREAM_CONTENT) this.rootGroup.add(subtreeGroup) const batches = this.batcher.getBatches(subtreeId) batches.forEach((batch: Batch) => { const batchRenderable = batch.renderObject + batchRenderable.layers.set(ObjectLayers.STREAM_CONTENT) subtreeGroup.add(batch.renderObject) if (batch.geometryType === GeometryType.MESH) { const mesh = batchRenderable as unknown as Mesh @@ -452,6 +463,7 @@ export default class SpeckleRenderer { private addDirectLights() { this.sun = new DirectionalLight(0xffffff, 5) this.sun.name = 'sun' + this.sun.layers.set(ObjectLayers.STREAM_CONTENT) this._scene.add(this.sun) this.sun.castShadow = true diff --git a/packages/viewer/src/modules/legacy/SelectionHelper.js b/packages/viewer/src/modules/legacy/SelectionHelper.js index da704fd02..35d87902e 100644 --- a/packages/viewer/src/modules/legacy/SelectionHelper.js +++ b/packages/viewer/src/modules/legacy/SelectionHelper.js @@ -1,5 +1,6 @@ import * as THREE from 'three' import EventEmitter from '../EventEmitter' +import { ObjectLayers } from '../SpeckleRenderer' /** * Selects and deselects user added objects in the scene. Emits the array of all intersected objects on click. @@ -19,6 +20,7 @@ export default class _SelectionHelper extends EventEmitter { this.raycaster.params.Line.threshold = 0.1 this.raycaster.params.Line2 = {} this.raycaster.params.Line2.threshold = 1 + this.raycaster.layers.set(ObjectLayers.PROPS) // optional param allows for raycasting against a subset of objects // this.subset = typeof _options !== 'undefined' && typeof _options.subset !== 'undefined' ? _options.subset : null; diff --git a/packages/viewer/src/modules/objects/SpeckleRaycaster.ts b/packages/viewer/src/modules/objects/SpeckleRaycaster.ts index 1a400f647..a7907b59f 100644 --- a/packages/viewer/src/modules/objects/SpeckleRaycaster.ts +++ b/packages/viewer/src/modules/objects/SpeckleRaycaster.ts @@ -1,10 +1,12 @@ import { Object3D, Raycaster } from 'three' +import { ObjectLayers } from '../SpeckleRenderer' export class SpeckleRaycaster extends Raycaster { public onObjectIntersectionTest: (object: Object3D) => void = null constructor(origin?, direction?, near = 0, far = Infinity) { super(origin, direction, near, far) + this.layers.set(ObjectLayers.STREAM_CONTENT) } public intersectObjects(objects, recursive = true, intersects = []) { diff --git a/packages/viewer/src/modules/pipeline/ColorPass.ts b/packages/viewer/src/modules/pipeline/ColorPass.ts index b83a3c845..3a477aa03 100644 --- a/packages/viewer/src/modules/pipeline/ColorPass.ts +++ b/packages/viewer/src/modules/pipeline/ColorPass.ts @@ -1,10 +1,17 @@ -import { Camera, Scene, Texture } from 'three' -import { RenderPass } from 'three/examples/jsm/postprocessing/RenderPass' -import { SpecklePass } from './SpecklePass' +import { Camera, Color, Material, Scene, Texture } from 'three' +import { BaseSpecklePass, SpecklePass } from './SpecklePass' -export class ColorPass extends RenderPass implements SpecklePass { - public constructor(scene: Scene, camera: Camera) { - super(scene, camera) +export class ColorPass extends BaseSpecklePass implements SpecklePass { + private camera: Camera + private scene: Scene + private overrideMaterial: Material = null + private _oldClearColor: Color = new Color() + private clearColor: Color = null + private clearAlpha = 0 + private clearDepth = true + + public constructor() { + super() } public get displayName(): string { return 'COLOR' @@ -12,4 +19,56 @@ export class ColorPass extends RenderPass implements SpecklePass { public get outputTexture(): Texture { return null } + + public update(scene: Scene, camera: Camera) { + this.camera = camera + this.scene = scene + } + + render(renderer, writeBuffer, readBuffer /*, deltaTime, maskActive */) { + const oldAutoClear = renderer.autoClear + renderer.autoClear = false + + let oldClearAlpha, oldOverrideMaterial + + if (this.overrideMaterial !== undefined) { + oldOverrideMaterial = this.scene.overrideMaterial + + this.scene.overrideMaterial = this.overrideMaterial + } + + if (this.clearColor) { + renderer.getClearColor(this._oldClearColor) + oldClearAlpha = renderer.getClearAlpha() + + renderer.setClearColor(this.clearColor, this.clearAlpha) + } + + if (this.clearDepth) { + renderer.clearDepth() + } + + this.applyLayers(this.camera) + + renderer.setRenderTarget(this.renderToScreen ? null : readBuffer) + + // TODO: Avoid using autoClear properties, see https://github.com/mrdoob/three.js/pull/15571#issuecomment-465669600 + if (this.clear) + renderer.clear( + renderer.autoClearColor, + renderer.autoClearDepth, + renderer.autoClearStencil + ) + renderer.render(this.scene, this.camera) + + if (this.clearColor) { + renderer.setClearColor(this._oldClearColor, oldClearAlpha) + } + + if (this.overrideMaterial !== undefined) { + this.scene.overrideMaterial = oldOverrideMaterial + } + + renderer.autoClear = oldAutoClear + } } diff --git a/packages/viewer/src/modules/pipeline/DepthPass.ts b/packages/viewer/src/modules/pipeline/DepthPass.ts index e0b4f3b5a..fa4ff4824 100644 --- a/packages/viewer/src/modules/pipeline/DepthPass.ts +++ b/packages/viewer/src/modules/pipeline/DepthPass.ts @@ -12,9 +12,8 @@ import { Texture, WebGLRenderTarget } from 'three' -import { Pass } from 'three/examples/jsm/postprocessing/Pass' import SpeckleDepthMaterial from '../materials/SpeckleDepthMaterial' -import { SpecklePass } from './SpecklePass' +import { BaseSpecklePass, SpecklePass } from './SpecklePass' export enum DepthType { PERSPECTIVE_DEPTH, @@ -26,7 +25,7 @@ export enum DepthSize { HALF } -export class DepthPass extends Pass implements SpecklePass { +export class DepthPass extends BaseSpecklePass implements SpecklePass { private renderTarget: WebGLRenderTarget private renderTargetHalf: WebGLRenderTarget private depthMaterial: SpeckleDepthMaterial = null @@ -134,6 +133,7 @@ export class DepthPass extends Pass implements SpecklePass { this.scene.overrideMaterial = this.depthMaterial renderer.shadowMap.enabled = false renderer.shadowMap.needsUpdate = false + this.applyLayers(this.camera) renderer.render(this.scene, this.camera) renderer.shadowMap.enabled = shadowmapEnabled renderer.shadowMap.needsUpdate = shadowmapNeedsUpdate diff --git a/packages/viewer/src/modules/pipeline/NormalsPass.ts b/packages/viewer/src/modules/pipeline/NormalsPass.ts index 771d0b165..d4aa86872 100644 --- a/packages/viewer/src/modules/pipeline/NormalsPass.ts +++ b/packages/viewer/src/modules/pipeline/NormalsPass.ts @@ -8,11 +8,10 @@ import { Texture, WebGLRenderTarget } from 'three' -import { Pass } from 'three/examples/jsm/postprocessing/Pass' import SpeckleNormalMaterial from '../materials/SpeckleNormalMaterial' -import { SpecklePass } from './SpecklePass' +import { BaseSpecklePass, SpecklePass } from './SpecklePass' -export class NormalsPass extends Pass implements SpecklePass { +export class NormalsPass extends BaseSpecklePass implements SpecklePass { private renderTarget: WebGLRenderTarget private normalsMaterial: SpeckleNormalMaterial = null private scene: Scene @@ -77,6 +76,7 @@ export class NormalsPass extends Pass implements SpecklePass { this.scene.overrideMaterial = this.normalsMaterial renderer.shadowMap.enabled = false renderer.shadowMap.needsUpdate = false + this.applyLayers(this.camera) renderer.render(this.scene, this.camera) renderer.shadowMap.enabled = shadowmapEnabled renderer.shadowMap.needsUpdate = shadowmapNeedsUpdate diff --git a/packages/viewer/src/modules/pipeline/Pipeline.ts b/packages/viewer/src/modules/pipeline/Pipeline.ts index f0d119092..ecf67f272 100644 --- a/packages/viewer/src/modules/pipeline/Pipeline.ts +++ b/packages/viewer/src/modules/pipeline/Pipeline.ts @@ -1,10 +1,10 @@ -import { Camera, Plane, Scene, Vector2, WebGLRenderer } from 'three' +import { Plane, Vector2, WebGLRenderer } from 'three' import { EffectComposer, Pass } from 'three/examples/jsm/postprocessing/EffectComposer.js' import Batcher from '../batching/Batcher' -import SpeckleRenderer from '../SpeckleRenderer' +import SpeckleRenderer, { ObjectLayers } from '../SpeckleRenderer' import { ApplySAOPass } from './ApplyAOPass' import { CopyOutputPass } from './CopyOutputPass' import { DepthPass, DepthSize, DepthType } from './DepthPass' @@ -220,17 +220,21 @@ export class Pipeline { this.composer.writeBuffer = null } - public configure(scene: Scene, camera: Camera) { + public configure() { this.depthPass = new DepthPass() this.normalsPass = new NormalsPass() this.dynamicAoPass = new DynamicSAOPass() - this.renderPass = new ColorPass(scene, camera) + this.renderPass = new ColorPass() this.applySaoPass = new ApplySAOPass() this.staticAoPass = new StaticAOPass() this.copyOutputPass = new CopyOutputPass() this.copyOutputPass.renderToScreen = true + this.depthPass.setLayers([ObjectLayers.STREAM_CONTENT]) + this.normalsPass.setLayers([ObjectLayers.STREAM_CONTENT]) + this.renderPass.setLayers([ObjectLayers.PROPS, ObjectLayers.STREAM_CONTENT]) + let restoreVisibility this.depthPass.onBeforeRender = () => { restoreVisibility = this._batcher.saveVisiblity() @@ -308,7 +312,7 @@ export class Pipeline { } public update(renderer: SpeckleRenderer) { - this.renderPass.camera = renderer.camera + this.renderPass.update(renderer.scene, renderer.camera) this.depthPass.update(renderer.scene, renderer.camera) this.dynamicAoPass.update(renderer.scene, renderer.camera) this.normalsPass.update(renderer.scene, renderer.camera) diff --git a/packages/viewer/src/modules/pipeline/SpecklePass.ts b/packages/viewer/src/modules/pipeline/SpecklePass.ts index f014fbec0..6d323bb7b 100644 --- a/packages/viewer/src/modules/pipeline/SpecklePass.ts +++ b/packages/viewer/src/modules/pipeline/SpecklePass.ts @@ -1,4 +1,6 @@ import { Camera, Plane, Scene, Texture } from 'three' +import { Pass } from 'three/examples/jsm/postprocessing/Pass' +import { ObjectLayers } from '../SpeckleRenderer' import { RenderType } from './Pipeline' export type InputColorTextureUniform = 'tDiffuse' @@ -17,9 +19,40 @@ export interface SpecklePass { setTexture?(uName: string, texture: Texture) setParams?(params: unknown) setClippingPlanes?(planes: Plane[]) + setLayers?(layers: ObjectLayers[]) } export interface SpeckleProgressivePass extends SpecklePass { setFrameIndex(index: number) setRenderType?(type: RenderType) } + +export abstract class BaseSpecklePass extends Pass implements SpecklePass { + protected layers: ObjectLayers[] = null + + constructor() { + super() + } + + get displayName(): string { + return 'BASE' + } + get outputTexture(): Texture { + return null + } + + public setLayers(layers: ObjectLayers[]) { + this.layers = layers + } + + protected applyLayers(camera: Camera) { + if (this.layers === null) { + camera.layers.enableAll() + return + } + camera.layers.disableAll() + this.layers.forEach((layer) => { + camera.layers.enable(layer) + }) + } +} From 62500996875cdd8e02627d79f5127142f637f35b Mon Sep 17 00:00:00 2001 From: AlexandruPopovici Date: Mon, 31 Oct 2022 16:47:21 +0200 Subject: [PATCH 41/54] Added more time between frames for the preview service. It seems to be generating the previews properly now --- packages/preview-service/routes/preview.js | 8 ++++++-- packages/viewer-sandbox/src/Sandbox.ts | 2 +- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/packages/preview-service/routes/preview.js b/packages/preview-service/routes/preview.js index de14e74c7..c55b9ffc7 100644 --- a/packages/preview-service/routes/preview.js +++ b/packages/preview-service/routes/preview.js @@ -29,11 +29,15 @@ async function pageFunction(objectUrl) { window.v.zoom(undefined, 0.95, false) await waitForAnimation(100) - // full 360. This needs testing with the new progressive pipeline for (let i = 0; i < 24; i++) { window.v.setView({ azimuth: Math.PI / 12, polar: 0 }, false) window.v.getRenderer().resetPipeline(true) - await waitForAnimation(1000) + /** Not sure what the frame time when running pupeteer is, but it's not 16ms. + * That's why we're allowing more time between frames than probably needed + * In a future update, we'll have the viewer signal when convergence is complete + * regradless of how many frames/time that takes + */ + await waitForAnimation(2500) ret.scr[i + ''] = await window.v.screenshot() } diff --git a/packages/viewer-sandbox/src/Sandbox.ts b/packages/viewer-sandbox/src/Sandbox.ts index 85d769347..07d0fcbff 100644 --- a/packages/viewer-sandbox/src/Sandbox.ts +++ b/packages/viewer-sandbox/src/Sandbox.ts @@ -61,8 +61,8 @@ export default class Sandbox { castShadow: true, intensity: 5, color: 0xffffff, - elevation: 1.33, azimuth: 0.75, + elevation: 1.33, radius: 0, indirectLightIntensity: 1.2 } From 048f1de3c9e26bb5be1d25b4937e16424efea96b Mon Sep 17 00:00:00 2001 From: AlexandruPopovici Date: Wed, 2 Nov 2022 10:16:07 +0200 Subject: [PATCH 42/54] Debuging bad max param --- packages/viewer-sandbox/src/Sandbox.ts | 99 ++++++++++++++++++- packages/viewer-sandbox/src/main.ts | 4 +- .../src/modules/pipeline/StaticAOPass.ts | 2 +- 3 files changed, 102 insertions(+), 3 deletions(-) diff --git a/packages/viewer-sandbox/src/Sandbox.ts b/packages/viewer-sandbox/src/Sandbox.ts index 07d0fcbff..bbede39b5 100644 --- a/packages/viewer-sandbox/src/Sandbox.ts +++ b/packages/viewer-sandbox/src/Sandbox.ts @@ -252,7 +252,104 @@ export default class Sandbox { title: 'Screenshot' }) screenshot.on('click', async () => { - console.warn(await this.viewer.screenshot()) + // console.warn(await this.viewer.screenshot()) + this.viewer.hideObjects( + [ + 'd0c1da9ee6dca24b2ce5a0dcac929bba', + '2d5b8a02dddde15c5d88e0f5adb26d90', + '3c086d63f9ec8eef67b4071f260a31dc', + '933e292e7c55dd083ddd80e3d4264fb5', + '33871108c2e6ee2deaf09226e191e389', + '34acd00132c576c9ccec05326db005a0', + 'ca7f371344c14130b4977910de457177', + '8a15ebff809996b4a54a9671674f9d82', + '9f181b38ec42e6aac2243d93d17f6290', + '90850bb5900b04a0ad3f2a0fc00b9dea', + '77c8f48d287e0a39485ea5d0e25e0760', + '2ec6b19932d70434c73c4b217a544cb9', + 'd78420715aef45dce9dd5d9df2efb9fd', + '80e59e3301f1ed7a59917636d8368eab', + '9f62d5a7c639260cd526ddd427086d31', + 'bd1ccc42212a9aaac803ac72b35e53de', + 'df01935c34b0b34f60e41376763000b4', + 'e41e6310c00d53305346e7c863b16076', + 'aeb8541e59717d7f7b985be1bd9d3c2c', + '154054257f2094a46129fdb09fc202d2', + '7c99e3828a96fbd5ea794c9996517c49', + 'f6f0fbb1177d6790ce5fccf460e82142', + '0a61a2e378b1ac64751e25dc8a30f2a5', + 'cc72f04be66aa3bf01f137e056658c39', + '3c1b7f96454a20e12a084f9b72a7eb9a', + '18a368993923f3b08b5be886252d799d', + '1b633a9b4c57f04c9509ea5fa4fa9bea', + '7419c6ae4cda06de6754d70ea5d1f00b', + 'ed7975192c92ec08475abb1f00d6c1a3', + '06d3eb307c93fe26305a9dc6686275d2', + '40c1b3170bf4254d96c75abe30334102', + '6272b36762238e93bcd90a998d745a5c', + '9aa660351afe6a8da7a2141deb0ca0dc', + '2c726a1513c67a6916c370f136cbb5d7', + '129aaf383e1a0e5c5bc282d84e98d025', + '8d26df55f42f0a7bffba76935379111c', + '5908ba1aec898bbd5a3af455bbafedf1', + 'f841cd136609a572a9547ff5f993d892', + '0bca01e0b4b33e151f832b820d140a49', + '004dd8fcd4cc059d805bf9ba32140158', + '264b09a99191fb9b26a4f1d018170ae9', + '2b82d0d17d419402f06f148cbadbe27b', + 'a511137a7997b7d049f3a6205d5c6067', + 'c73988b267632b0643003c24c957fc7f', + '22be75a1b272605c922f8214356e777a', + '6960442387221968072d9e1d7ff6dd17', + '9b13c9236459b0961f9f078dce802585', + '5165d828b6546a342862644332a629ea', + 'e794f07cdafffdce83b7466b39055e3b', + '71c514a012f0dea6959f8831a12a7e2b', + '6a4d6ec7e1ab4c9de49328e261cd5cce', + 'afd0d559f53048e207b2ee0414daa4c2', + 'a15ce6a9ab7211c3f080039b58c026c9', + 'e2e77f23f0f3bfadd124577f5195f74b', + '0e55912cffcab3871a0d3c8fd66ac8c9', + 'c2e29dabb928d8b0f17ab454040c7bca', + 'be39301ebb6eb7b382530b1ab301d924', + '14f91d947a9c9ac921abbff008d0bf69', + 'cbdaf984d11e039c0c16e1661b000002', + '1d21b5d3e8573a6aec196d54bbacdd36', + '3fc9117a2b10af66b62fc20a0d1746cd', + '824bc34ae86317c9b7e216dbe543bdf1', + 'ea338b1ef4518e00d424bd8060a612de', + '4aaff96888e14e30268b8befb99cd32a', + 'da5fd648bf1d096660b1093ae9943a0c', + 'a0c730d9ce179ce66bfe469eec0d7ce2', + '69d5cde7e59dc66c5bf5469942dfbcb8', + '720465b8343ea257a9b31094f30d79e5', + 'e63494bd4b5687e6067a02708584ca39', + 'bb34922b2162e979ba0a9bb212c869b8', + '2ef029249422b0fe24859094b0bada40', + '30358584b2bbf6b9bc2fecad361ee310', + '67e4495051bf3e6b7915763bb26e64cd', + '60b59b8586308098284ed3e02c041dd6', + '56dbaf612bfc7a87595cc1f6eb7df349', + 'cdb5ce431d4b5cfd7cd2c567f3248117', + 'd135a886b6f7dbbda642a5ba00ed5246', + '299dba6eb18d74ab9a5611acbb691ab3', + '426fcd488d42cf2788eace0178b2316d', + '1afdfb290bd8dc1061eeb30d8dadcb20', + '81c706e841718c6fbd7cd7ae54e60826', + 'fd447c87ca5bdf39c624056ef1e0e6ad', + 'bd067857e20be63391b42f57e8f47dac', + 'f4e6a5b8ee37066638514f9e6b801558', + '1f1b413799b5e1a86bd44d98d37985dc', + 'c18d318764aba688743171ea5ce0f715', + '5a1464ed66785b3db7fee16516a72e0c', + '5ea1957470434d024a7c429c0397e35f', + 'd7f72f0b7e4d26ce4b37638d88ed41ab', + '28cebf87f08efa15223a012f55ba3ee6', + '8f4e916a68eb86c84ce2746f426af9d2' + ], + 'ui-vis', + false + ) }) const rotate = this.tabs.pages[0].addButton({ diff --git a/packages/viewer-sandbox/src/main.ts b/packages/viewer-sandbox/src/main.ts index 349a1cb3a..c8be6b533 100644 --- a/packages/viewer-sandbox/src/main.ts +++ b/packages/viewer-sandbox/src/main.ts @@ -82,7 +82,7 @@ sandbox.makeFilteringUI() await sandbox.loadUrl( // 'https://speckle.xyz/streams/da9e320dad/commits/5388ef24b8?c=%5B-7.66134,10.82932,6.41935,-0.07739,-13.88552,1.8697,0,1%5D' // Revit sample house (good for bim-like stuff with many display meshes) - 'https://speckle.xyz/streams/da9e320dad/commits/5388ef24b8' + // 'https://speckle.xyz/streams/da9e320dad/commits/5388ef24b8' // 'Super' heavy revit shit // 'https://speckle.xyz/streams/e6f9156405/commits/0694d53bb5' // IFC building (good for a tree based structure) @@ -126,4 +126,6 @@ await sandbox.loadUrl( // 'https://latest.speckle.dev/streams/2d19273d31/commits/9ceb423feb' // Luis sphere // 'https://speckle.xyz/streams/b85d53c3b4/commits/b47f21b707' + // Building AO params + 'https://latest.speckle.dev/streams/0dd74866d0/commits/317e210afa' ) diff --git a/packages/viewer/src/modules/pipeline/StaticAOPass.ts b/packages/viewer/src/modules/pipeline/StaticAOPass.ts index c84717a81..7625e7267 100644 --- a/packages/viewer/src/modules/pipeline/StaticAOPass.ts +++ b/packages/viewer/src/modules/pipeline/StaticAOPass.ts @@ -54,7 +54,7 @@ export const DefaultStaticAoPassParams = { kernelSize: 16, bias: 0.01, minDistance: 0, - maxDistance: 0.008 + maxDistance: 1 //0.008 } export class StaticAOPass extends Pass implements SpeckleProgressivePass { From 0c99573bc68dfc377c8fc99ba5433410f54e0ed0 Mon Sep 17 00:00:00 2001 From: Iain Sproat <68657+iainsproat@users.noreply.github.com> Date: Wed, 2 Nov 2022 10:01:19 +0000 Subject: [PATCH 43/54] Fixes liveness and readiness checks to prevent CSRF error message (#1169) - provides content-type header - check that status code is 200 --- utils/helm/speckle-server/templates/server/deployment.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/utils/helm/speckle-server/templates/server/deployment.yml b/utils/helm/speckle-server/templates/server/deployment.yml index ab3e3defd..510957713 100644 --- a/utils/helm/speckle-server/templates/server/deployment.yml +++ b/utils/helm/speckle-server/templates/server/deployment.yml @@ -70,7 +70,7 @@ spec: command: - node - -e - - require('request')('http://localhost:3000/graphql?query={serverInfo{version}}', (e,r,b) => process.exit(b.toLowerCase().includes('error'))) + - require('request')({headers: {'Content-Type': 'application/json'}, uri: 'http://localhost:3000/graphql?query={serverInfo{version}}', method: 'GET' }, (e,r,b) => process.exit(r.statusCode != 200 || b.toLowerCase().includes('error'))) readinessProbe: initialDelaySeconds: 5 @@ -80,7 +80,7 @@ spec: command: - node - -e - - require('request')('http://localhost:3000/graphql?query={serverInfo{version}}', (e,r,b) => process.exit(b.toLowerCase().includes('error'))) + - require('request')({headers: {'Content-Type': 'application/json'}, uri: 'http://localhost:3000/graphql?query={serverInfo{version}}', method: 'GET' }, (e,r,b) => process.exit(r.statusCode != 200 || b.toLowerCase().includes('error'))) env: - name: CANONICAL_URL From df250d616da1777ac64c1c57199debfd5cca1258 Mon Sep 17 00:00:00 2001 From: Iain Sproat <68657+iainsproat@users.noreply.github.com> Date: Wed, 2 Nov 2022 10:40:03 +0000 Subject: [PATCH 44/54] Fixes broken helm template by adding quotation marks around liveness probe command (#1171) --- utils/helm/speckle-server/templates/server/deployment.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/utils/helm/speckle-server/templates/server/deployment.yml b/utils/helm/speckle-server/templates/server/deployment.yml index 510957713..f34f5e465 100644 --- a/utils/helm/speckle-server/templates/server/deployment.yml +++ b/utils/helm/speckle-server/templates/server/deployment.yml @@ -70,7 +70,7 @@ spec: command: - node - -e - - require('request')({headers: {'Content-Type': 'application/json'}, uri: 'http://localhost:3000/graphql?query={serverInfo{version}}', method: 'GET' }, (e,r,b) => process.exit(r.statusCode != 200 || b.toLowerCase().includes('error'))) + - "require('request')({headers: {'Content-Type': 'application/json'}, uri: 'http://localhost:3000/graphql?query={serverInfo{version}}', method: 'GET' }, (e,r,b) => process.exit(r.statusCode != 200 || b.toLowerCase().includes('error')))" readinessProbe: initialDelaySeconds: 5 @@ -80,7 +80,7 @@ spec: command: - node - -e - - require('request')({headers: {'Content-Type': 'application/json'}, uri: 'http://localhost:3000/graphql?query={serverInfo{version}}', method: 'GET' }, (e,r,b) => process.exit(r.statusCode != 200 || b.toLowerCase().includes('error'))) + - "require('request')({headers: {'Content-Type': 'application/json'}, uri: 'http://localhost:3000/graphql?query={serverInfo{version}}', method: 'GET' }, (e,r,b) => process.exit(r.statusCode != 200 || b.toLowerCase().includes('error')))" env: - name: CANONICAL_URL From fc2534e9b47c4e277c34e7155c34786682c51bc6 Mon Sep 17 00:00:00 2001 From: Iain Sproat <68657+iainsproat@users.noreply.github.com> Date: Wed, 2 Nov 2022 11:45:15 +0000 Subject: [PATCH 45/54] Always build, except when branch (not main) does not have PR (#1167) * Always build, except when branch (not main) does not have PR * Logic fix * Should fail the step * Debug * More circleci debugging * More and more circleci debugging * More more more debugging * Attempt to explicitly exit * :facepalm: the script required a passing line in order to exit non-zero --- .circleci/config.yml | 26 ++++++++++---------------- 1 file changed, 10 insertions(+), 16 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 4f9c3efed..0b535cb0b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -14,14 +14,6 @@ workflows: - get-version: filters: *filters-allow-all - - build-approval: - type: approval - filters: &filters-ignore-main-branch-all-tags - branches: - ignore: main - tags: - ignore: /.*/ - - pre-commit: filters: *filters-allow-all @@ -32,51 +24,48 @@ workflows: requires: - test-server - get-version - - build-approval - docker-build-frontend: filters: *filters-build requires: - get-version - - build-approval - docker-build-webhooks: filters: *filters-build requires: - get-version - test-server - - build-approval - docker-build-file-imports: filters: *filters-build requires: - get-version - test-server - - build-approval - docker-build-previews: filters: *filters-build requires: - get-version - test-server - - build-approval - docker-build-test-container: filters: *filters-build requires: - get-version - test-server - - build-approval - docker-build-monitor-container: filters: *filters-build requires: - get-version - - build-approval - publish-approval: type: approval - filters: *filters-ignore-main-branch-all-tags + filters: &filters-ignore-main-branch-or-all-tags + branches: + ignore: main + tags: + ignore: /.*/ - docker-publish-server: context: &docker-hub-context @@ -336,6 +325,11 @@ jobs: - attach_workspace: at: /tmp/ci/workspace - run: cat workspace/env-vars >> $BASH_ENV + - run: + name: 'Check if should proceed' + command: | + [[ "${CIRCLE_BRANCH}" != "main" && -z "${CIRCLE_PULL_REQUEST}" ]] && echo "Should not build, stopping" && circleci-agent step halt && exit 1 + echo "proceeding" - setup_remote_docker: # a weird issue with yarn installing packages throwing EPERM errors # this fixes it From c06650c47545e91c002e97bc8fa287e3d655047e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gerg=C5=91=20Jedlicska?= <57442769+gjedlicska@users.noreply.github.com> Date: Wed, 2 Nov 2022 12:57:46 +0100 Subject: [PATCH 46/54] Update to new specklepy (#1173) * Publish images for all branches but limit tagging * only tag 'latest' and '2' when 'SHOULD_PUBLISH' variable is 'true' * Publishing helm chart should check for `SHOULD_PUBLISH` * Move blocking step to publish-helm chart, and allow images to be published * Pin python requirements and bump to latest versions * Fix EOL whitespace * use valid version for psycopg2-binary (the clue is in the 2!) * fix(fileimports): add exception printing to file imports * fix(fileimports): bump specklepy version move to a specklepy version that contains a fix for send without writing to disk Co-authored-by: Iain Sproat <68657+iainsproat@users.noreply.github.com> --- .../fileimport-service/obj/import_file.py | 102 ++++++++++-------- packages/fileimport-service/requirements.txt | 2 +- .../fileimport-service/stl/import_file.py | 52 +++++---- 3 files changed, 89 insertions(+), 67 deletions(-) diff --git a/packages/fileimport-service/obj/import_file.py b/packages/fileimport-service/obj/import_file.py index 853440538..a63aa4f80 100644 --- a/packages/fileimport-service/obj/import_file.py +++ b/packages/fileimport-service/obj/import_file.py @@ -1,53 +1,58 @@ - +import sys, os import json from specklepy.objects import Base from specklepy.objects.other import RenderMaterial -from specklepy.objects.geometry import Mesh, Point, Line +from specklepy.objects.geometry import Mesh from specklepy.transports.server import ServerTransport from specklepy.api.client import SpeckleClient -from specklepy.api.credentials import get_default_account from specklepy.api import operations - -import sys, os - from obj_file import ObjFile -TMP_RESULTS_PATH = '/tmp/import_result.json' -DEFAULT_BRANCH = 'uploads' + +TMP_RESULTS_PATH = "/tmp/import_result.json" +DEFAULT_BRANCH = "uploads" + def convert_material(obj_mat): speckle_mat = RenderMaterial() - speckle_mat.name = obj_mat['name'] - if 'diffuse' in obj_mat: - argb = [1,] + obj_mat['diffuse'] - speckle_mat.diffuse = int.from_bytes([int(val * 255) for val in argb], byteorder="big", signed=True) - if 'dissolved' in obj_mat: - speckle_mat.opacity = obj_mat['dissolved'] + speckle_mat.name = obj_mat["name"] + if "diffuse" in obj_mat: + argb = [ + 1, + ] + obj_mat["diffuse"] + speckle_mat.diffuse = int.from_bytes( + [int(val * 255) for val in argb], byteorder="big", signed=True + ) + if "dissolved" in obj_mat: + speckle_mat.opacity = obj_mat["dissolved"] return speckle_mat + def import_obj(): - file_path, user_id, stream_id, branch_name, commit_message = sys.argv[1:] - print(f'ImportOBJ argv[1:]: {sys.argv[1:]}') + file_path, _, stream_id, branch_name, commit_message = sys.argv[1:] + print(f"ImportOBJ argv[1:]: {sys.argv[1:]}") # Parse input obj = ObjFile(file_path) - print(f'Parsed obj with {len(obj.faces)} faces ({len(obj.vertices) * 3} vertices)') + print(f"Parsed obj with {len(obj.faces)} faces ({len(obj.vertices) * 3} vertices)") speckle_root = Base() - speckle_root['@objects'] = [] + speckle_root["@objects"] = [] for objname in obj.objects: - print(f' Converting {objname}...') + print(f" Converting {objname}...") speckle_obj = Base() speckle_obj.name = objname - speckle_obj['@displayValue'] = [] - speckle_root['@objects'].append(speckle_obj) - + speckle_obj["@displayValue"] = [] + speckle_root["@objects"].append(speckle_obj) + for obj_mesh in obj.objects[objname]: - speckle_vertices = [coord for point in obj_mesh['vertices'] for coord in point] + speckle_vertices = [ + coord for point in obj_mesh["vertices"] for coord in point + ] speckle_faces = [] - for obj_face in obj_mesh['faces']: + for obj_face in obj_mesh["faces"]: if len(obj_face) == 3: speckle_faces.append(0) elif len(obj_face) == 4: @@ -57,64 +62,75 @@ def import_obj(): speckle_faces.extend(obj_face) has_vertex_colors = False - for vc in obj_mesh['vertex_colors']: + for vc in obj_mesh["vertex_colors"]: if vc is not None: has_vertex_colors = True colors = [] if has_vertex_colors: - for vc in obj_mesh['vertex_colors']: + for vc in obj_mesh["vertex_colors"]: if vc is None: r, g, b = (1.0, 1.0, 1.0) else: r, g, b = vc argb = (1.0, r, g, b) - color = int.from_bytes([int(val * 255) for val in argb], byteorder="big", signed=True) + color = int.from_bytes( + [int(val * 255) for val in argb], byteorder="big", signed=True + ) colors.append(color) speckle_mesh = Mesh( vertices=speckle_vertices, faces=speckle_faces, colors=colors, - textureCoordinates=[] + textureCoordinates=[], ) - obj_material = obj_mesh['material'] + obj_material = obj_mesh["material"] if obj_material: - speckle_mesh['renderMaterial'] = convert_material(obj_material) + speckle_mesh["renderMaterial"] = convert_material(obj_material) - speckle_obj['@displayValue'].append(speckle_mesh) + speckle_obj["@displayValue"].append(speckle_mesh) # Commit - client = SpeckleClient(host=os.getenv('SPECKLE_SERVER_URL', 'localhost:3000'), use_ssl=False) - client.authenticate(os.environ['USER_TOKEN']) + client = SpeckleClient( + host=os.getenv("SPECKLE_SERVER_URL", "localhost:3000"), use_ssl=False + ) + client.authenticate_with_token(os.environ["USER_TOKEN"]) if not client.branch.get(stream_id, branch_name): - client.branch.create(stream_id, branch_name, 'File upload branch' if branch_name == 'uploads' else '') + client.branch.create( + stream_id, + branch_name, + "File upload branch" if branch_name == "uploads" else "", + ) transport = ServerTransport(client=client, stream_id=stream_id) - id = operations.send(base=speckle_root, transports=[transport], use_default_cache=False) + id = operations.send( + base=speckle_root, transports=[transport], use_default_cache=False + ) commit_id = client.commit.create( stream_id=stream_id, object_id=id, branch_name=(branch_name or DEFAULT_BRANCH), - message=(commit_message or 'OBJ file upload'), - source_application='OBJ' + message=(commit_message or "OBJ file upload"), + source_application="OBJ", ) return commit_id -if __name__ == '__main__': +if __name__ == "__main__": + from pathlib import Path + try: commit_id = import_obj() if not commit_id: raise Exception("Can't create commit") - results = {'success': True, 'commitId': commit_id} + results = {"success": True, "commitId": commit_id} except Exception as ex: - print('ERROR: ' + str(ex)) - results = {'success': False, 'error': str(ex)} + print("ERROR: " + str(ex)) + results = {"success": False, "error": str(ex)} - with open(TMP_RESULTS_PATH, 'w') as f: - json.dump(results, f) + Path(TMP_RESULTS_PATH).write_text(json.dumps(results)) diff --git a/packages/fileimport-service/requirements.txt b/packages/fileimport-service/requirements.txt index 7b3349251..f7e1614c8 100644 --- a/packages/fileimport-service/requirements.txt +++ b/packages/fileimport-service/requirements.txt @@ -1,2 +1,2 @@ numpy-stl==2.17.1 -specklepy==2.9.0 +specklepy==2.9.1 diff --git a/packages/fileimport-service/stl/import_file.py b/packages/fileimport-service/stl/import_file.py index 8de5c4f95..ac7d8740c 100644 --- a/packages/fileimport-service/stl/import_file.py +++ b/packages/fileimport-service/stl/import_file.py @@ -1,46 +1,50 @@ - import json import stl -from specklepy.objects.geometry import Mesh, Point +from specklepy.objects.geometry import Mesh from specklepy.transports.server import ServerTransport from specklepy.api.client import SpeckleClient -from specklepy.api.credentials import get_default_account from specklepy.api import operations import sys, os -TMP_RESULTS_PATH = '/tmp/import_result.json' -DEFAULT_BRANCH = 'uploads' +TMP_RESULTS_PATH = "/tmp/import_result.json" +DEFAULT_BRANCH = "uploads" + def import_stl(): - file_path, user_id, stream_id, branch_name, commit_message = sys.argv[1:] - print(f'ImportSTL argv[1:]: {sys.argv[1:]}') + file_path, _, stream_id, branch_name, commit_message = sys.argv[1:] + print(f"ImportSTL argv[1:]: {sys.argv[1:]}") # Parse input stl_mesh = stl.mesh.Mesh.from_file(file_path) - print(f'Parsed mesh with {stl_mesh.points.shape[0]} faces ({stl_mesh.points.shape[0] * 3} vertices)') + print( + f"Parsed mesh with {stl_mesh.points.shape[0]} faces ({stl_mesh.points.shape[0] * 3} vertices)" + ) # Construct speckle obj vertices = stl_mesh.points.flatten().tolist() faces = [] for i in range(stl_mesh.points.shape[0]): - faces.extend([0, 3*i, 3*i+1, 3*i+2]) + faces.extend([0, 3 * i, 3 * i + 1, 3 * i + 2]) speckle_mesh = Mesh( - vertices=vertices, - faces=faces, - colors=[], - textureCoordinates=[] + vertices=vertices, faces=faces, colors=[], textureCoordinates=[] ) - print('Constructed Speckle Mesh object') + print("Constructed Speckle Mesh object") # Commit - client = SpeckleClient(host=os.getenv('SPECKLE_SERVER_URL', 'localhost:3000'), use_ssl=False) - client.authenticate(os.environ['USER_TOKEN']) + client = SpeckleClient( + host=os.getenv("SPECKLE_SERVER_URL", "localhost:3000"), use_ssl=False + ) + client.authenticate_with_token(os.environ["USER_TOKEN"]) if not client.branch.get(stream_id, branch_name): - client.branch.create(stream_id, branch_name, 'File upload branch' if branch_name == 'uploads' else '') + client.branch.create( + stream_id, + branch_name, + "File upload branch" if branch_name == "uploads" else "", + ) transport = ServerTransport(client=client, stream_id=stream_id) id = operations.send( @@ -53,20 +57,22 @@ def import_stl(): stream_id=stream_id, object_id=id, branch_name=(branch_name or DEFAULT_BRANCH), - message=(commit_message or 'STL file upload'), - source_application='STL' + message=(commit_message or "STL file upload"), + source_application="STL", ) return commit_id -if __name__ == '__main__': +if __name__ == "__main__": + from pathlib import Path + try: commit_id = import_stl() - results = {'success': True, 'commitId': commit_id} + results = {"success": True, "commitId": commit_id} except Exception as ex: results = {'success': False, 'error': str(ex)} print(ex) - with open(TMP_RESULTS_PATH, 'w') as f: - json.dump(results, f) + print(results) + Path(TMP_RESULTS_PATH).write_text(json.dumps(results)) From de9beccd229f5ad651bfdad7cb332ccf86568bdb Mon Sep 17 00:00:00 2001 From: Iain Sproat <68657+iainsproat@users.noreply.github.com> Date: Wed, 2 Nov 2022 17:16:53 +0000 Subject: [PATCH 47/54] Helm test is deployed as a job (#1174) - this allows it to be identified in alerting more easily --- .../templates/tests/deployment.yml | 78 ++++++++++--------- 1 file changed, 41 insertions(+), 37 deletions(-) diff --git a/utils/helm/speckle-server/templates/tests/deployment.yml b/utils/helm/speckle-server/templates/tests/deployment.yml index e29205717..25b977c26 100644 --- a/utils/helm/speckle-server/templates/tests/deployment.yml +++ b/utils/helm/speckle-server/templates/tests/deployment.yml @@ -1,49 +1,53 @@ {{- if .Values.helm_test_enabled }} -apiVersion: v1 -kind: Pod + +apiVersion: batch/v1 +kind: Job metadata: - name: "speckle-test-deployment" + name: "speckle-test" namespace: {{ .Values.namespace }} annotations: - "helm.sh/hook": test + helm.sh/hook: test labels: {{ include "test.labels" . | indent 4 }} spec: - containers: - - name: test-deployment - image: speckle/speckle-test-deployment:{{ .Values.docker_image_tag }} - env: - - name: SPECKLE_SERVER - value: https://{{ .Values.domain }} - - name: SERVER_VERSION - value: {{ .Values.docker_image_tag }} - resources: - requests: - cpu: {{ .Values.test.requests.cpu }} - memory: {{ .Values.test.requests.memory }} - limits: - cpu: {{ .Values.test.limits.cpu }} - memory: {{ .Values.test.limits.memory }} + backoffLimit: 1 + parallelism: 1 + completions: 1 + template: + spec: + containers: + - name: test-deployment + image: speckle/speckle-test-deployment:{{ .Values.docker_image_tag }} + env: + - name: SPECKLE_SERVER + value: https://{{ .Values.domain }} + - name: SERVER_VERSION + value: {{ .Values.docker_image_tag }} + resources: + requests: + cpu: {{ .Values.test.requests.cpu }} + memory: {{ .Values.test.requests.memory }} + limits: + cpu: {{ .Values.test.limits.cpu }} + memory: {{ .Values.test.limits.memory }} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + privileged: false + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 20000 + restartPolicy: Never securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - privileged: false - readOnlyRootFilesystem: true runAsNonRoot: true runAsUser: 20000 + runAsGroup: 30000 + seccompProfile: + type: RuntimeDefault - restartPolicy: Never - - securityContext: - runAsNonRoot: true - runAsUser: 20000 - runAsGroup: 30000 - seccompProfile: - type: RuntimeDefault - - {{- if .Values.test.serviceAccount.create }} - serviceAccountName: {{ include "test.name" $ }} - {{- end }} + {{- if .Values.test.serviceAccount.create }} + serviceAccountName: {{ include "test.name" $ }} + {{- end }} {{- end }} From 7c6a09e4a5f6514d73fa9c2531053a96d1c1e771 Mon Sep 17 00:00:00 2001 From: AlexandruPopovici Date: Fri, 4 Nov 2022 14:50:28 +0200 Subject: [PATCH 48/54] Removed some debug code --- packages/viewer-sandbox/src/Sandbox.ts | 99 +------------------------- 1 file changed, 1 insertion(+), 98 deletions(-) diff --git a/packages/viewer-sandbox/src/Sandbox.ts b/packages/viewer-sandbox/src/Sandbox.ts index bbede39b5..07d0fcbff 100644 --- a/packages/viewer-sandbox/src/Sandbox.ts +++ b/packages/viewer-sandbox/src/Sandbox.ts @@ -252,104 +252,7 @@ export default class Sandbox { title: 'Screenshot' }) screenshot.on('click', async () => { - // console.warn(await this.viewer.screenshot()) - this.viewer.hideObjects( - [ - 'd0c1da9ee6dca24b2ce5a0dcac929bba', - '2d5b8a02dddde15c5d88e0f5adb26d90', - '3c086d63f9ec8eef67b4071f260a31dc', - '933e292e7c55dd083ddd80e3d4264fb5', - '33871108c2e6ee2deaf09226e191e389', - '34acd00132c576c9ccec05326db005a0', - 'ca7f371344c14130b4977910de457177', - '8a15ebff809996b4a54a9671674f9d82', - '9f181b38ec42e6aac2243d93d17f6290', - '90850bb5900b04a0ad3f2a0fc00b9dea', - '77c8f48d287e0a39485ea5d0e25e0760', - '2ec6b19932d70434c73c4b217a544cb9', - 'd78420715aef45dce9dd5d9df2efb9fd', - '80e59e3301f1ed7a59917636d8368eab', - '9f62d5a7c639260cd526ddd427086d31', - 'bd1ccc42212a9aaac803ac72b35e53de', - 'df01935c34b0b34f60e41376763000b4', - 'e41e6310c00d53305346e7c863b16076', - 'aeb8541e59717d7f7b985be1bd9d3c2c', - '154054257f2094a46129fdb09fc202d2', - '7c99e3828a96fbd5ea794c9996517c49', - 'f6f0fbb1177d6790ce5fccf460e82142', - '0a61a2e378b1ac64751e25dc8a30f2a5', - 'cc72f04be66aa3bf01f137e056658c39', - '3c1b7f96454a20e12a084f9b72a7eb9a', - '18a368993923f3b08b5be886252d799d', - '1b633a9b4c57f04c9509ea5fa4fa9bea', - '7419c6ae4cda06de6754d70ea5d1f00b', - 'ed7975192c92ec08475abb1f00d6c1a3', - '06d3eb307c93fe26305a9dc6686275d2', - '40c1b3170bf4254d96c75abe30334102', - '6272b36762238e93bcd90a998d745a5c', - '9aa660351afe6a8da7a2141deb0ca0dc', - '2c726a1513c67a6916c370f136cbb5d7', - '129aaf383e1a0e5c5bc282d84e98d025', - '8d26df55f42f0a7bffba76935379111c', - '5908ba1aec898bbd5a3af455bbafedf1', - 'f841cd136609a572a9547ff5f993d892', - '0bca01e0b4b33e151f832b820d140a49', - '004dd8fcd4cc059d805bf9ba32140158', - '264b09a99191fb9b26a4f1d018170ae9', - '2b82d0d17d419402f06f148cbadbe27b', - 'a511137a7997b7d049f3a6205d5c6067', - 'c73988b267632b0643003c24c957fc7f', - '22be75a1b272605c922f8214356e777a', - '6960442387221968072d9e1d7ff6dd17', - '9b13c9236459b0961f9f078dce802585', - '5165d828b6546a342862644332a629ea', - 'e794f07cdafffdce83b7466b39055e3b', - '71c514a012f0dea6959f8831a12a7e2b', - '6a4d6ec7e1ab4c9de49328e261cd5cce', - 'afd0d559f53048e207b2ee0414daa4c2', - 'a15ce6a9ab7211c3f080039b58c026c9', - 'e2e77f23f0f3bfadd124577f5195f74b', - '0e55912cffcab3871a0d3c8fd66ac8c9', - 'c2e29dabb928d8b0f17ab454040c7bca', - 'be39301ebb6eb7b382530b1ab301d924', - '14f91d947a9c9ac921abbff008d0bf69', - 'cbdaf984d11e039c0c16e1661b000002', - '1d21b5d3e8573a6aec196d54bbacdd36', - '3fc9117a2b10af66b62fc20a0d1746cd', - '824bc34ae86317c9b7e216dbe543bdf1', - 'ea338b1ef4518e00d424bd8060a612de', - '4aaff96888e14e30268b8befb99cd32a', - 'da5fd648bf1d096660b1093ae9943a0c', - 'a0c730d9ce179ce66bfe469eec0d7ce2', - '69d5cde7e59dc66c5bf5469942dfbcb8', - '720465b8343ea257a9b31094f30d79e5', - 'e63494bd4b5687e6067a02708584ca39', - 'bb34922b2162e979ba0a9bb212c869b8', - '2ef029249422b0fe24859094b0bada40', - '30358584b2bbf6b9bc2fecad361ee310', - '67e4495051bf3e6b7915763bb26e64cd', - '60b59b8586308098284ed3e02c041dd6', - '56dbaf612bfc7a87595cc1f6eb7df349', - 'cdb5ce431d4b5cfd7cd2c567f3248117', - 'd135a886b6f7dbbda642a5ba00ed5246', - '299dba6eb18d74ab9a5611acbb691ab3', - '426fcd488d42cf2788eace0178b2316d', - '1afdfb290bd8dc1061eeb30d8dadcb20', - '81c706e841718c6fbd7cd7ae54e60826', - 'fd447c87ca5bdf39c624056ef1e0e6ad', - 'bd067857e20be63391b42f57e8f47dac', - 'f4e6a5b8ee37066638514f9e6b801558', - '1f1b413799b5e1a86bd44d98d37985dc', - 'c18d318764aba688743171ea5ce0f715', - '5a1464ed66785b3db7fee16516a72e0c', - '5ea1957470434d024a7c429c0397e35f', - 'd7f72f0b7e4d26ce4b37638d88ed41ab', - '28cebf87f08efa15223a012f55ba3ee6', - '8f4e916a68eb86c84ce2746f426af9d2' - ], - 'ui-vis', - false - ) + console.warn(await this.viewer.screenshot()) }) const rotate = this.tabs.pages[0].addButton({ From dc3fd1dfb173c9a27afdce336cfc3017cf8e6b5f Mon Sep 17 00:00:00 2001 From: AlexandruPopovici Date: Fri, 4 Nov 2022 15:01:09 +0200 Subject: [PATCH 49/54] Removed min and max distance from the sandbox UI since they are no longer used as params --- packages/viewer-sandbox/src/Sandbox.ts | 40 +++++++++++++------------- packages/viewer-sandbox/src/main.ts | 4 +-- 2 files changed, 22 insertions(+), 22 deletions(-) diff --git a/packages/viewer-sandbox/src/Sandbox.ts b/packages/viewer-sandbox/src/Sandbox.ts index 07d0fcbff..12987a4ce 100644 --- a/packages/viewer-sandbox/src/Sandbox.ts +++ b/packages/viewer-sandbox/src/Sandbox.ts @@ -499,27 +499,27 @@ export default class Sandbox { this.viewer.getRenderer().pipelineOptions = Sandbox.pipelineParams this.viewer.requestRender() }) - staticAoFolder - .addInput(Sandbox.pipelineParams.staticAoParams, 'minDistance', { - min: 0, - max: 100, - step: 0.000001 - }) - .on('change', () => { - this.viewer.getRenderer().pipelineOptions = Sandbox.pipelineParams - this.viewer.requestRender() - }) + // staticAoFolder + // .addInput(Sandbox.pipelineParams.staticAoParams, 'minDistance', { + // min: 0, + // max: 100, + // step: 0.000001 + // }) + // .on('change', () => { + // this.viewer.getRenderer().pipelineOptions = Sandbox.pipelineParams + // this.viewer.requestRender() + // }) - staticAoFolder - .addInput(Sandbox.pipelineParams.staticAoParams, 'maxDistance', { - min: 0, - max: 100, - step: 0.000001 - }) - .on('change', () => { - this.viewer.getRenderer().pipelineOptions = Sandbox.pipelineParams - this.viewer.requestRender() - }) + // staticAoFolder + // .addInput(Sandbox.pipelineParams.staticAoParams, 'maxDistance', { + // min: 0, + // max: 100, + // step: 0.000001 + // }) + // .on('change', () => { + // this.viewer.getRenderer().pipelineOptions = Sandbox.pipelineParams + // this.viewer.requestRender() + // }) staticAoFolder .addInput(Sandbox.pipelineParams.staticAoParams, 'kernelRadius', { min: 0, diff --git a/packages/viewer-sandbox/src/main.ts b/packages/viewer-sandbox/src/main.ts index a40858e77..dd4c0bb89 100644 --- a/packages/viewer-sandbox/src/main.ts +++ b/packages/viewer-sandbox/src/main.ts @@ -97,7 +97,7 @@ await sandbox.loadUrl( // AutoCAD // 'https://latest.speckle.dev/streams/3ed8357f29/commits/d10f2af1ce' //Blizzard world - 'https://latest.speckle.dev/streams/0c6ad366c4/commits/aa1c393aec' + // 'https://latest.speckle.dev/streams/0c6ad366c4/commits/aa1c393aec' //Car // 'https://latest.speckle.dev/streams/17d2e25a97/commits/6b6cf3d43e' // Jonathon's @@ -127,7 +127,7 @@ await sandbox.loadUrl( // Luis sphere // 'https://speckle.xyz/streams/b85d53c3b4/commits/b47f21b707' // Building AO params - // 'https://latest.speckle.dev/streams/0dd74866d0/commits/317e210afa' + 'https://latest.speckle.dev/streams/0dd74866d0/commits/317e210afa' // Murder Cube // 'https://latest.speckle.dev/streams/c1faab5c62/commits/7f0c4d2fc1/' // Classroom From acc2e9688d4c6e9923059e8cea3f47d074183055 Mon Sep 17 00:00:00 2001 From: AlexandruPopovici Date: Fri, 4 Nov 2022 15:28:39 +0200 Subject: [PATCH 50/54] Removed the maxDist calculation from the loop --- .../materials/shaders/speckle-static-ao-generate-frag.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/viewer/src/modules/materials/shaders/speckle-static-ao-generate-frag.ts b/packages/viewer/src/modules/materials/shaders/speckle-static-ao-generate-frag.ts index 72d1400e1..09121fb8a 100644 --- a/packages/viewer/src/modules/materials/shaders/speckle-static-ao-generate-frag.ts +++ b/packages/viewer/src/modules/materials/shaders/speckle-static-ao-generate-frag.ts @@ -265,6 +265,7 @@ export const speckleStaticAoGenerateFrag = /* glsl */ ` float occlusion = 0.0; float kernelSize_ws = computeKernelSize(-viewPosition.z, kernelRadius); float div = float( KERNEL_SIZE); + float maxDist = kernelSize_ws / (cameraFar - cameraNear); for ( int i = 0; i < KERNEL_SIZE; i ++ ) { vec3 sampleVector = kernelMatrix * kernel[ i ]; // reorient sample vector in view space vec3 samplePoint = viewPosition + ( sampleVector * kernelSize_ws ); // calculate sample point @@ -274,7 +275,7 @@ export const speckleStaticAoGenerateFrag = /* glsl */ ` float realDepth = getLinearDepth( samplePointUv ); // get linear depth from depth texture float sampleDepth = viewZToOrthographicDepth( samplePoint.z + bias, cameraNear, cameraFar ); // compute linear depth of the sample view Z value float delta = sampleDepth - realDepth; - if ( delta > minDistance && delta < kernelSize_ws / (cameraFar - cameraNear) ) { // if fragment is before sample point, increase occlusion + if ( delta > 0. && delta < maxDist ) { // if fragment is before sample point, increase occlusion occlusion += 1.0; } } From 8c067b1ad7a15a6d6f6e90ea40065a3233f65f17 Mon Sep 17 00:00:00 2001 From: Iain Sproat <68657+iainsproat@users.noreply.github.com> Date: Fri, 4 Nov 2022 18:21:37 +0000 Subject: [PATCH 51/54] fix(preview-service prometheus): additional buckets in prometheus histogram (#1184) All previews were in the 600 to infinity bucket of the histogram, rendering it effectively useless. This PR adds additional larger buckets at 1200 and 1800 ranges to provide better granularity. --- packages/preview-service/bg_service/prometheusMetrics.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/preview-service/bg_service/prometheusMetrics.js b/packages/preview-service/bg_service/prometheusMetrics.js index 6de7db95d..4e3c84f73 100644 --- a/packages/preview-service/bg_service/prometheusMetrics.js +++ b/packages/preview-service/bg_service/prometheusMetrics.js @@ -100,7 +100,7 @@ module.exports = { metricDuration: new prometheusClient.Histogram({ name: 'speckle_server_operation_duration', help: 'Summary of the operation durations in seconds', - buckets: [0.5, 1, 5, 10, 30, 60, 300, 600], + buckets: [0.5, 1, 5, 10, 30, 60, 300, 600, 1200, 1800], labelNames: ['op'] }), From 071a9588ddbf1dc7bad14520747dce4feb69021e Mon Sep 17 00:00:00 2001 From: Iain Sproat <68657+iainsproat@users.noreply.github.com> Date: Wed, 9 Nov 2022 12:06:50 +0000 Subject: [PATCH 52/54] ci(snyk): snyk vulnerability scan of yarn package.json (#1192) * ci(snyk): snyk vulnerability scan of yarn package.json * Run with strict-out-of-sync=false * only runs for main branch to prevent pollution of Snyk's dashboard with branches --- .circleci/config.yml | 41 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 0b535cb0b..72b6d5557 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,10 +1,22 @@ version: 2.1 +orbs: + snyk: snyk/snyk@1.4.0 + workflows: version: 2 test-build: jobs: + - vulnerability-scan: + context: &snyk-context + - snyk + filters: + branches: + only: + - main + - hotfix* + - test-server: filters: &filters-allow-all tags: @@ -315,6 +327,35 @@ jobs: # path: packages/server/coverage/lcov-report # destination: package/server/coverage + vulnerability-scan: + # snyk can undertake most types of scans through GitHub integration + # which does not require integration with the CI + # but it is not possible to scan npm/yarn package.json + # because it requires node_modules + # therefore this scanning has to be triggered via the cli + docker: &docker-image + - image: cimg/python:3.9.15-node + resource_class: small + working_directory: *work-dir + steps: + - checkout + - restore_cache: + name: Restore Yarn Package Cache + keys: + - yarn-packages-server-{{ checksum "yarn.lock" }} + - run: + name: Install Dependencies + command: yarn + - save_cache: + name: Save Yarn Package Cache + key: yarn-packages-server-{{ checksum "yarn.lock" }} + paths: + - .yarn/cache + - .yarn/unplugged + - snyk/scan: + additional-arguments: --yarn-workspaces --strict-out-of-sync=false + fail-on-issues: false + docker-build: &build-job docker: &docker-image - image: cimg/python:3.9.15-node From 3d6653f73b51d092a73e4d7683f9912dc92295bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gerg=C5=91=20Jedlicska?= <57442769+gjedlicska@users.noreply.github.com> Date: Wed, 9 Nov 2022 13:23:32 +0100 Subject: [PATCH 53/54] hotfix/2.9.2 (#1175) * Update to new specklepy (#1173) * Publish images for all branches but limit tagging * only tag 'latest' and '2' when 'SHOULD_PUBLISH' variable is 'true' * Publishing helm chart should check for `SHOULD_PUBLISH` * Move blocking step to publish-helm chart, and allow images to be published * Pin python requirements and bump to latest versions * Fix EOL whitespace * use valid version for psycopg2-binary (the clue is in the 2!) * fix(fileimports): add exception printing to file imports * fix(fileimports): bump specklepy version move to a specklepy version that contains a fix for send without writing to disk Co-authored-by: Iain Sproat <68657+iainsproat@users.noreply.github.com> * Fixes liveness and readiness checks to prevent CSRF error message (#1169) - provides content-type header - check that status code is 200 * Fixes broken helm template by adding quotation marks around liveness probe command (#1171) * fix(server activities): make sure the stream events are properly dispatched * feat(server webhooks): add scheduled orphaned webhook cleanup * test(server webhooks): add test to webhook cleanup service * feat(server webhooks): drop foreign key reference for webhooks schema to streams * refactor(server req context): refactor req context to have the ip attribute for all requests * feat(server objects rest api): add ratelimits to objects rest api endpoints * fix(server rest api): properly handle returning 419 Co-authored-by: Iain Sproat <68657+iainsproat@users.noreply.github.com> --- .../modules/activitystream/services/index.js | 2 +- .../server/modules/auth/strategies/local.js | 18 +---- .../server/modules/core/rest/diffDownload.js | 9 +++ .../server/modules/core/rest/diffUpload.js | 9 +++ packages/server/modules/core/rest/download.js | 17 ++++ packages/server/modules/core/rest/upload.js | 10 +++ .../modules/core/services/ratelimits.js | 55 +++++++++---- packages/server/modules/index.js | 3 +- packages/server/modules/shared/index.js | 3 +- packages/server/modules/shared/utils/ip.js | 15 ++++ packages/server/modules/webhooks/index.js | 0 packages/server/modules/webhooks/index.ts | 27 +++++++ .../20221104104921_webhooks_drop_stream_fk.ts | 16 ++++ .../modules/webhooks/services/cleanup.ts | 15 ++++ .../modules/webhooks/services/webhooks.js | 2 + .../modules/webhooks/tests/cleanup.spec.ts | 81 +++++++++++++++++++ 16 files changed, 247 insertions(+), 35 deletions(-) create mode 100644 packages/server/modules/shared/utils/ip.js delete mode 100644 packages/server/modules/webhooks/index.js create mode 100644 packages/server/modules/webhooks/index.ts create mode 100644 packages/server/modules/webhooks/migrations/20221104104921_webhooks_drop_stream_fk.ts create mode 100644 packages/server/modules/webhooks/services/cleanup.ts create mode 100644 packages/server/modules/webhooks/tests/cleanup.spec.ts diff --git a/packages/server/modules/activitystream/services/index.js b/packages/server/modules/activitystream/services/index.js index 76c47c62b..9b0901bc9 100644 --- a/packages/server/modules/activitystream/services/index.js +++ b/packages/server/modules/activitystream/services/index.js @@ -40,7 +40,7 @@ module.exports = { data: info } } - dispatchStreamEvent({ + await dispatchStreamEvent({ streamId, event: actionType, eventPayload: webhooksPayload diff --git a/packages/server/modules/auth/strategies/local.js b/packages/server/modules/auth/strategies/local.js index 2d84acfde..80896560e 100644 --- a/packages/server/modules/auth/strategies/local.js +++ b/packages/server/modules/auth/strategies/local.js @@ -12,6 +12,7 @@ const { finalizeInvitedServerRegistration, resolveAuthRedirectPath } = require('@/modules/serverinvites/services/inviteProcessingService') +const { getIpFromRequest } = require('@/modules/shared/utils/ip') module.exports = async (app, session, sessionAppId, finalizeAuth) => { const strategy = { @@ -57,21 +58,8 @@ module.exports = async (app, session, sessionAppId, finalizeAuth) => { if (!req.body.password) throw new Error('Password missing') const user = req.body - user.ip = req.headers['cf-connecting-ip'] || req.connection.remoteAddress || '' - const ignorePrefixes = [ - '192.168.', - '10.', - '127.', - '172.1', - '172.2', - '172.3', - '::' - ] - for (const ipPrefix of ignorePrefixes) - if (user.ip.startsWith(ipPrefix)) { - delete user.ip - break - } + const ip = getIpFromRequest(req) + if (ip) user.ip = ip if ( user.ip && !(await respectsLimits({ action: 'USER_CREATE', source: user.ip })) diff --git a/packages/server/modules/core/rest/diffDownload.js b/packages/server/modules/core/rest/diffDownload.js index 7f9387d59..f6fd1b0e0 100644 --- a/packages/server/modules/core/rest/diffDownload.js +++ b/packages/server/modules/core/rest/diffDownload.js @@ -7,6 +7,9 @@ const { contextMiddleware } = require('@/modules/shared') const { validatePermissionsReadStream } = require('./authUtils') const { SpeckleObjectsStream } = require('./speckleObjectsStream') const { getObjectsStream } = require('../services/objects') +const { + rejectsRequestWithRatelimitStatusIfNeeded +} = require('@/modules/core/services/ratelimits') const { pipeline, PassThrough } = require('stream') @@ -14,6 +17,12 @@ module.exports = (app) => { app.options('/api/getobjects/:streamId', cors()) app.post('/api/getobjects/:streamId', cors(), contextMiddleware, async (req, res) => { + const rejected = await rejectsRequestWithRatelimitStatusIfNeeded({ + action: 'POST /api/getobjects/:streamId', + req, + res + }) + if (rejected) return rejected const hasStreamAccess = await validatePermissionsReadStream( req.params.streamId, req diff --git a/packages/server/modules/core/rest/diffUpload.js b/packages/server/modules/core/rest/diffUpload.js index 38df741e0..fb33d8fe9 100644 --- a/packages/server/modules/core/rest/diffUpload.js +++ b/packages/server/modules/core/rest/diffUpload.js @@ -5,6 +5,9 @@ const debug = require('debug') const { contextMiddleware } = require('@/modules/shared') const { validatePermissionsWriteStream } = require('./authUtils') +const { + rejectsRequestWithRatelimitStatusIfNeeded +} = require('@/modules/core/services/ratelimits') const { hasObjects } = require('../services/objects') @@ -12,6 +15,12 @@ module.exports = (app) => { app.options('/api/diff/:streamId', cors()) app.post('/api/diff/:streamId', cors(), contextMiddleware, async (req, res) => { + const rejected = await rejectsRequestWithRatelimitStatusIfNeeded({ + action: 'POST /api/diff/:streamId', + req, + res + }) + if (rejected) return rejected const hasStreamAccess = await validatePermissionsWriteStream( req.params.streamId, req diff --git a/packages/server/modules/core/rest/download.js b/packages/server/modules/core/rest/download.js index af71d0eb2..644ffea62 100644 --- a/packages/server/modules/core/rest/download.js +++ b/packages/server/modules/core/rest/download.js @@ -9,6 +9,9 @@ const { validatePermissionsReadStream } = require('./authUtils') const { getObject, getObjectChildrenStream } = require('../services/objects') const { SpeckleObjectsStream } = require('./speckleObjectsStream') const { pipeline, PassThrough } = require('stream') +const { + rejectsRequestWithRatelimitStatusIfNeeded +} = require('@/modules/core/services/ratelimits') module.exports = (app) => { app.options('/objects/:streamId/:objectId', cors()) @@ -18,6 +21,13 @@ module.exports = (app) => { cors(), contextMiddleware, async (req, res) => { + const rejected = await rejectsRequestWithRatelimitStatusIfNeeded({ + action: 'GET /objects/:streamId/:objectId', + req, + res + }) + if (rejected) return rejected + const hasStreamAccess = await validatePermissionsReadStream( req.params.streamId, req @@ -85,6 +95,13 @@ module.exports = (app) => { cors(), contextMiddleware, async (req, res) => { + const rejected = await rejectsRequestWithRatelimitStatusIfNeeded({ + action: 'GET /objects/:streamId/:objectId/single', + req, + res + }) + if (rejected) return rejected + const hasStreamAccess = await validatePermissionsReadStream( req.params.streamId, req diff --git a/packages/server/modules/core/rest/upload.js b/packages/server/modules/core/rest/upload.js index 2448c340e..68ca0dd50 100644 --- a/packages/server/modules/core/rest/upload.js +++ b/packages/server/modules/core/rest/upload.js @@ -8,6 +8,9 @@ const { contextMiddleware } = require('@/modules/shared') const { validatePermissionsWriteStream } = require('./authUtils') const { createObjectsBatched } = require('../services/objects') +const { + rejectsRequestWithRatelimitStatusIfNeeded +} = require('@/modules/core/services/ratelimits') const MAX_FILE_SIZE = 50 * 1024 * 1024 @@ -15,6 +18,13 @@ module.exports = (app) => { app.options('/objects/:streamId', cors()) app.post('/objects/:streamId', cors(), contextMiddleware, async (req, res) => { + const rejected = await rejectsRequestWithRatelimitStatusIfNeeded({ + action: 'POST /objects/:streamId', + req, + res + }) + if (rejected) return rejected + const hasStreamAccess = await validatePermissionsWriteStream( req.params.streamId, req diff --git a/packages/server/modules/core/services/ratelimits.js b/packages/server/modules/core/services/ratelimits.js index 7b5043bd0..d5cad08ed 100644 --- a/packages/server/modules/core/services/ratelimits.js +++ b/packages/server/modules/core/services/ratelimits.js @@ -25,7 +25,13 @@ const LIMITS = { BRANCHES: parseInt(process.env.LIMIT_BRANCHES) || 1000, // per stream TOKENS: parseInt(process.env.LIMIT_TOKENS) || 1000, // per user ACTIVE_SUBSCRIPTIONS: parseInt(process.env.LIMIT_ACTIVE_SUBSCRIPTIONS) || 100, // per user - ACTIVE_CONNECTIONS: parseInt(process.env.LIMIT_ACTIVE_CONNECTIONS) || 100 // per source ip + ACTIVE_CONNECTIONS: parseInt(process.env.LIMIT_ACTIVE_CONNECTIONS) || 100, // per source ip + + 'POST /api/getobjects/:streamId': 200, // for 1 minute + 'POST /api/diff/:streamId': 200, // for 1 minute + 'POST /objects/:streamId': 200, // for 1 minute + 'GET /objects/:streamId/:objectId': 200, // for 1 minute + 'GET /objects/:streamId/:objectId/single': 200 // for 1 minute } const LIMIT_INTERVAL = { @@ -42,7 +48,13 @@ const LIMIT_INTERVAL = { BRANCHES: 0, TOKENS: 0, ACTIVE_SUBSCRIPTIONS: 0, - ACTIVE_CONNECTIONS: 0 + ACTIVE_CONNECTIONS: 0, + + 'POST /api/getobjects/:streamId': 60, + 'POST /api/diff/:streamId': 60, + 'POST /objects/:streamId': 60, + 'GET /objects/:streamId/:objectId': 60, + 'GET /objects/:streamId/:objectId/single': 60 } const rateLimitedCache = {} @@ -74,22 +86,31 @@ async function shouldRateLimitNext({ action, source }) { return shouldRateLimit } +// returns true if the action is fine, false if it should be blocked because of exceeding limit +async function respectsLimits({ action, source }) { + const rateLimitKey = `${action} ${source}` + const promise = shouldRateLimitNext({ action, source }).then((shouldRateLimit) => { + if (shouldRateLimit) rateLimitedCache[rateLimitKey] = true + else delete rateLimitedCache[rateLimitKey] + }) + if (rateLimitedCache[rateLimitKey]) { + await promise + } + + if (rateLimitedCache[rateLimitKey]) limitsReached.labels(action).inc() + return !rateLimitedCache[rateLimitKey] +} + +async function rejectsRequestWithRatelimitStatusIfNeeded({ action, req, res }) { + const source = req.context.userId || req.context.ip + if (!(await respectsLimits({ action, source }))) + return res.status(429).set('X-Speckle-Meditation', 'https://http.cat/429').send({ + err: 'You are sending too many requests. You have been rate limited. Please try again later.' + }) +} module.exports = { LIMITS, LIMIT_INTERVAL, - - // returns true if the action is fine, false if it should be blocked because of exceeding limit - async respectsLimits({ action, source }) { - const rateLimitKey = `${action} ${source}` - const promise = shouldRateLimitNext({ action, source }).then((shouldRateLimit) => { - if (shouldRateLimit) rateLimitedCache[rateLimitKey] = true - else delete rateLimitedCache[rateLimitKey] - }) - if (rateLimitedCache[rateLimitKey]) { - await promise - } - - if (rateLimitedCache[rateLimitKey]) limitsReached.labels(action).inc() - return !rateLimitedCache[rateLimitKey] - } + respectsLimits, + rejectsRequestWithRatelimitStatusIfNeeded } diff --git a/packages/server/modules/index.js b/packages/server/modules/index.js index bedfdb12a..a3b617b41 100644 --- a/packages/server/modules/index.js +++ b/packages/server/modules/index.js @@ -55,7 +55,8 @@ async function getSpeckleModules() { './blobstorage', './notifications', './activitystream', - './accessrequests' + './accessrequests', + './webhooks' ] for (const dir of moduleDirs) { diff --git a/packages/server/modules/shared/index.js b/packages/server/modules/shared/index.js index 0af24a2bd..0923471d7 100644 --- a/packages/server/modules/shared/index.js +++ b/packages/server/modules/shared/index.js @@ -5,6 +5,7 @@ const { ForbiddenError, ApolloError } = require('apollo-server-express') const { RedisPubSub } = require('graphql-redis-subscriptions') const { buildRequestLoaders } = require('@/modules/core/loaders') const { validateToken } = require(`@/modules/core/services/tokens`) +const { getIpFromRequest } = require('@/modules/shared/utils/ip') const StreamPubsubEvents = Object.freeze({ UserStreamAdded: 'USER_STREAM_ADDED', @@ -49,7 +50,7 @@ function addLoadersToCtx(ctx) { async function buildContext({ req, connection }) { // Parsing auth info const ctx = await contextApiTokenHelper({ req, connection }) - + ctx.ip = getIpFromRequest(req) // Adding request data loaders return addLoadersToCtx(ctx) } diff --git a/packages/server/modules/shared/utils/ip.js b/packages/server/modules/shared/utils/ip.js new file mode 100644 index 000000000..f8db74a66 --- /dev/null +++ b/packages/server/modules/shared/utils/ip.js @@ -0,0 +1,15 @@ +const getIpFromRequest = (req) => { + let ip + try { + ip = req.headers['cf-connecting-ip'] || req.ip || req.connection.remoteAddress || '' + } catch { + ip = '' + } + const ignorePrefixes = ['192.168.', '10.', '127.', '172.1', '172.2', '172.3', '::'] + + for (const ipPrefix of ignorePrefixes) + if (ip.startsWith(ipPrefix) || ip === '') return null + return ip +} + +module.exports = { getIpFromRequest } diff --git a/packages/server/modules/webhooks/index.js b/packages/server/modules/webhooks/index.js deleted file mode 100644 index e69de29bb..000000000 diff --git a/packages/server/modules/webhooks/index.ts b/packages/server/modules/webhooks/index.ts new file mode 100644 index 000000000..396eda435 --- /dev/null +++ b/packages/server/modules/webhooks/index.ts @@ -0,0 +1,27 @@ +import cron from 'node-cron' +import { SpeckleModule } from '@/modules/shared/helpers/typeHelper' +import { modulesDebug } from '@/modules/shared/utils/logger' +import { scheduleExecution } from '@/modules/core/services/taskScheduler' +import { cleanOrphanedWebhookConfigs } from '@/modules/webhooks/services/cleanup' + +const webhooksDebug = modulesDebug.extend('activities') + +const scheduleWebhookCleanup = () => { + const cronExpression = '0 4 * * 1' + return scheduleExecution(cronExpression, 'weeklyWebhookCleanup', async () => { + webhooksDebug('Starting weekly webhooks cleanup') + await cleanOrphanedWebhookConfigs() + webhooksDebug('Finished cleanup') + }) +} + +let scheduledTask: cron.ScheduledTask | null = null + +export const init: SpeckleModule['init'] = () => { + modulesDebug('🎣 Init webhooks module') + scheduledTask = scheduleWebhookCleanup() +} + +export const shutdown: SpeckleModule['shutdown'] = () => { + if (scheduledTask) scheduledTask.stop() +} diff --git a/packages/server/modules/webhooks/migrations/20221104104921_webhooks_drop_stream_fk.ts b/packages/server/modules/webhooks/migrations/20221104104921_webhooks_drop_stream_fk.ts new file mode 100644 index 000000000..4597f2823 --- /dev/null +++ b/packages/server/modules/webhooks/migrations/20221104104921_webhooks_drop_stream_fk.ts @@ -0,0 +1,16 @@ +import { Knex } from 'knex' + +const TABLE_NAME = 'webhooks_config' + +export async function up(knex: Knex): Promise { + await knex.schema.alterTable(TABLE_NAME, (table) => { + // dropping the foreign key and not adding a new one + // if we still had the constraint, the late triggered event ie stream_delete hooks + // would not find the webhook configs + table.dropForeign('streamId') + }) +} + +export async function down() { + return +} diff --git a/packages/server/modules/webhooks/services/cleanup.ts b/packages/server/modules/webhooks/services/cleanup.ts new file mode 100644 index 000000000..125c8244b --- /dev/null +++ b/packages/server/modules/webhooks/services/cleanup.ts @@ -0,0 +1,15 @@ +import knex from '@/db/knex' + +export async function cleanOrphanedWebhookConfigs() { + await knex.raw( + // i know im using a where in here, but this is used as background operation + ` + delete from webhooks_config + where id in ( + select wh.id from webhooks_config wh + left join streams st on wh."streamId" = st.id + where st.id is null + ) + ` + ) +} diff --git a/packages/server/modules/webhooks/services/webhooks.js b/packages/server/modules/webhooks/services/webhooks.js index 5a82387d5..07df06fd7 100644 --- a/packages/server/modules/webhooks/services/webhooks.js +++ b/packages/server/modules/webhooks/services/webhooks.js @@ -107,6 +107,8 @@ module.exports = { } } + // with this select, we must have the streamid available on the webhook config, + // even when the stream is deleted, to dispatch the stream deleted webhook events const { rows } = await knex.raw( ` SELECT * FROM webhooks_config WHERE "streamId" = ? diff --git a/packages/server/modules/webhooks/tests/cleanup.spec.ts b/packages/server/modules/webhooks/tests/cleanup.spec.ts new file mode 100644 index 000000000..94bde26c3 --- /dev/null +++ b/packages/server/modules/webhooks/tests/cleanup.spec.ts @@ -0,0 +1,81 @@ +import knex from '@/db/knex' +import { createStream } from '@/modules/core/services/streams' +import { createUser } from '@/modules/core/services/users' +import { cleanOrphanedWebhookConfigs } from '@/modules/webhooks/services/cleanup' +import { truncateTables } from '@/test/hooks' +import { expect } from 'chai' +import crs from 'crypto-random-string' + +const WEBHOOKS_CONFIG_TABLE = 'webhooks_config' +const WEBHOOKS_EVENTS_TABLE = 'webhooks_events' + +const WebhooksConfig = () => knex(WEBHOOKS_CONFIG_TABLE) +const randomId = () => crs({ length: 10 }) + +const countWebhooks = async () => { + const [{ count }] = await WebhooksConfig().count() + return parseInt(count as string) +} + +describe('Webhooks cleanup @webhooks', () => { + before(async () => { + await truncateTables([WEBHOOKS_CONFIG_TABLE, WEBHOOKS_EVENTS_TABLE]) + }) + + it('Cleans orphaned webhook configs', async () => { + const webhookConfig = { + id: randomId(), + streamId: randomId(), + url: 'foobar', + description: 'test_hook', + triggers: { + // eslint-disable-next-line camelcase + stream_update: true + } + } + await WebhooksConfig().insert(webhookConfig) + expect(await countWebhooks()).to.equal(1) + await cleanOrphanedWebhookConfigs() + expect(await countWebhooks()).to.equal(0) + }) + + it('Cleans orphans, leaves live ones intact', async () => { + const ownerId = await createUser({ + name: 'User', + email: 'user@gmail.com', + password: 'jdsadjsadasfdsa' + }) + const streamId = await createStream({ + name: 'foo', + description: 'bar', + ownerId + }) + + const webhookConfigs = [ + { + id: randomId(), + streamId: randomId(), + url: 'foobar', + description: 'test_hook', + triggers: { + // eslint-disable-next-line camelcase + stream_update: true + } + }, + { + id: randomId(), + streamId, + url: 'foobar', + description: 'test_hook', + triggers: { + // eslint-disable-next-line camelcase + stream_update: true + } + } + ] + await Promise.all(webhookConfigs.map((c) => WebhooksConfig().insert(c))) + expect(await countWebhooks()).to.equal(2) + await cleanOrphanedWebhookConfigs() + expect(await countWebhooks()).to.equal(1) + }) +}) From f0524b53dbf49a754a38471322271905f949a307 Mon Sep 17 00:00:00 2001 From: Kristaps Fabians Geikins Date: Tue, 15 Nov 2022 12:30:13 +0200 Subject: [PATCH 54/54] fix: various security alert fixes (#1202) * fix: some extra param validation for some API endpoints * fix(server): potentially leaking internal error details * fix: secure session cookie for ssl servers * fix(server): fixing tests --- packages/preview-service/routes/preview.js | 6 ++++++ packages/server/modules/auth/errors/index.ts | 6 ++++++ packages/server/modules/auth/rest/index.js | 21 ++++++++++++++++--- packages/server/modules/auth/strategies.js | 6 +++++- packages/server/modules/blobstorage/index.js | 8 +++++++ .../modules/shared/helpers/envHelper.ts | 7 +++++++ 6 files changed, 50 insertions(+), 4 deletions(-) create mode 100644 packages/server/modules/auth/errors/index.ts diff --git a/packages/preview-service/routes/preview.js b/packages/preview-service/routes/preview.js index c55b9ffc7..8f9d03291 100644 --- a/packages/preview-service/routes/preview.js +++ b/packages/preview-service/routes/preview.js @@ -126,6 +126,12 @@ async function getScreenshot(objectUrl) { } router.get('/:streamId/:objectId', async function (req, res) { + const safeParamRgx = /^[\w]+$/i + const { streamId, objectId } = req.params || {} + if (!safeParamRgx.test(streamId) || !safeParamRgx.test(objectId)) { + return res.status(400).json({ error: 'Invalid streamId or objectId!' }) + } + const objectUrl = `http://127.0.0.1:3001/streams/${req.params.streamId}/objects/${req.params.objectId}` /* let authToken = '' diff --git a/packages/server/modules/auth/errors/index.ts b/packages/server/modules/auth/errors/index.ts new file mode 100644 index 000000000..89050725d --- /dev/null +++ b/packages/server/modules/auth/errors/index.ts @@ -0,0 +1,6 @@ +import { BaseError } from '@/modules/shared/errors' + +export class InvalidAccessCodeRequestError extends BaseError { + static code = 'INVALID_ACCESS_CODE_REQUEST' + static defaultMessage = 'An issue occurred while generating an access code for an app' +} diff --git a/packages/server/modules/auth/rest/index.js b/packages/server/modules/auth/rest/index.js index 302a70594..391beb280 100644 --- a/packages/server/modules/auth/rest/index.js +++ b/packages/server/modules/auth/rest/index.js @@ -13,6 +13,8 @@ const { const { validateToken, revokeTokenById } = require(`@/modules/core/services/tokens`) const { revokeRefreshToken } = require(`@/modules/auth/services/apps`) const { validateScopes } = require(`@/modules/shared`) +const { InvalidAccessCodeRequestError } = require('@/modules/auth/errors') +const { ForbiddenError } = require('apollo-server-errors') // TODO: Secure these endpoints! module.exports = (app) => { @@ -24,14 +26,17 @@ module.exports = (app) => { try { const appId = req.query.appId const app = await getApp({ id: appId }) - if (!app) throw new Error('App does not exist.') + + if (!app) throw new InvalidAccessCodeRequestError('App does not exist.') const challenge = req.query.challenge const userToken = req.query.token + if (!challenge) throw new InvalidAccessCodeRequestError('Missing challenge') + if (!userToken) throw new InvalidAccessCodeRequestError('Missing token') // 1. Validate token const { valid, scopes, userId } = await validateToken(userToken) - if (!valid) throw new Error('Invalid token') + if (!valid) throw new InvalidAccessCodeRequestError('Invalid token') // 2. Validate token scopes await validateScopes(scopes, 'tokens:write') @@ -41,7 +46,17 @@ module.exports = (app) => { } catch (err) { sentry({ err }) debug('speckle:error')(err) - return res.status(400).send(err.message) + + if ( + err instanceof InvalidAccessCodeRequestError || + err instanceof ForbiddenError + ) { + return res.status(400).send(err.message) + } else { + return res + .status(500) + .send('Something went wrong while processing your request') + } } }) diff --git a/packages/server/modules/auth/strategies.js b/packages/server/modules/auth/strategies.js index 618f91b02..38d4ddb9f 100644 --- a/packages/server/modules/auth/strategies.js +++ b/packages/server/modules/auth/strategies.js @@ -7,6 +7,7 @@ const passport = require('passport') const sentry = require('@/logging/sentryHelper') const { createAuthorizationCode } = require('./services/apps') +const { isSSLServer } = require('@/modules/shared/helpers/envHelper') /** * TODO: Get rid of session entirely, we don't use it for the app and it's not really necessary for the auth flow, so it only complicates things @@ -24,7 +25,10 @@ module.exports = async (app) => { secret: process.env.SESSION_SECRET, saveUninitialized: false, resave: false, - cookie: { maxAge: 1000 * 60 * 3 } // 3 minutes + cookie: { + maxAge: 1000 * 60 * 3, // 3 minutes + secure: isSSLServer() + } }) /** diff --git a/packages/server/modules/blobstorage/index.js b/packages/server/modules/blobstorage/index.js index f5df6e8b4..965fff74b 100644 --- a/packages/server/modules/blobstorage/index.js +++ b/packages/server/modules/blobstorage/index.js @@ -31,6 +31,8 @@ const { getFileSizeLimit } = require('@/modules/blobstorage/services') +const { isArray } = require('lodash') + const { NotFoundError, ResourceMismatch, @@ -176,6 +178,12 @@ exports.init = async (app) => { allowAnonymousUsersOnPublicStreams ]), async (req, res) => { + if (!isArray(req.body)) { + return res + .status(400) + .json({ error: 'An array of blob IDs expected in the body.' }) + } + const bq = await getAllStreamBlobIds({ streamId: req.params.streamId }) const unknownBlobIds = req.body.filter( (id) => bq.findIndex((bInfo) => bInfo.id === id) === -1 diff --git a/packages/server/modules/shared/helpers/envHelper.ts b/packages/server/modules/shared/helpers/envHelper.ts index 3a6626c3e..fba084f22 100644 --- a/packages/server/modules/shared/helpers/envHelper.ts +++ b/packages/server/modules/shared/helpers/envHelper.ts @@ -56,3 +56,10 @@ export function shouldDisableNotificationsConsumption() { process.env.DISABLE_NOTIFICATIONS_CONSUMPTION || 'false' ) } + +/** + * Check whether we're running an SSL server + */ +export function isSSLServer() { + return /^https:\/\//.test(getBaseUrl()) +}