Skip to content

Commit

Permalink
feat(volume-rendering): picking, depth testing, and blending improvem…
Browse files Browse the repository at this point in the history 
…ents (#589)

* feat: add a mode override to VR layers
this allows to run second passes in different modes in the background, without affecting the UI value

* feat: run a second rendering pass on VR layers for picking

* feat: detect camera move in perspective panel

* feat: allow render context to know camera move

* feat: only two pass render VR pick when no camera movement

* refactor: simplify gl state setting in render loop for VR

* feat: only redraw if VR present as only VR uses redraw

* temp: second pass in single draw call progress

* temp: establish more shader control

* fix: show on VR color

* fix: correct setting of chunk translation

* feat: add context tracking of continous camera motion

* feat: link perspective panel to context camera tracking
Also only redraw if volume rendering is present after camera motion

* fix: rename camera movement in layers

* feat: mark rotation in panel as continuous

* feat: mark rendered data panel actions as camera move

* feat: mark slice rotations as continous camera

* refactor: clearer gl state tracking for vr on/max

* refactor: interface shader uniforms and reduce duplication

* fix: set new source outside of loop, ensure mode override can't get stuck on

* refactor: unify naming of camera motion in contexts

* fix: make unvarying const

* refactor: cleaner integration of new camera tracking

* fix: logic for when VR picking is available

* fix: correctly draw VR to new VR buffer in max/normal hybrid

* fix: OIT blend VR layer and other transparent layers after new buffer made

* refactor: normalize naming across files

* refactor: clearer buffer interaction during VR rendering

* fix: correctly set back to state and buffer during VR if a second pass or histogram is used

* refactor: re-use shader uniform setting function in max and non max VR

* refactor: fix typo in var name

* chore: fix formatting

* fix: turn off pre-depth testing in VR to allow in shader test instead

* feat: remove unused epth shader

* fix: enable required depth test for second pass max projection

* fix: allow wireframe mode in VR to work again

* feat: allow 8x VR downsampling

* refactor: remove unused "needToCleanUpVolumeRendering" variable
This always had the same value as "hasVolumeRendering", so "hasVolumeRendering" is used instead
  • Loading branch information
@seankmartin
seankmartin authored Jul 24, 2024
1 parent 5c8744d commit 4e378b4
Show file tree
Hide file tree
Showing 3 changed files with 348 additions and 164 deletions.
192 changes: 109 additions & 83 deletions src/perspective_view/panel.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -196,17 +196,17 @@ v4f_fragColor = vec4(accum.rgb / accum.a, revealage);
`);
}

// Copy the depth from opaque pass to the depth buffer for OIT.
// This copy is required because the OIT depth buffer might be
// smaller than the main depth buffer.
function defineDepthCopyShader(builder: ShaderBuilder) {
function defineTransparentToTransparentCopyShader(builder: ShaderBuilder) {
builder.addOutputBuffer("vec4", "v4f_fragData0", 0);
builder.addOutputBuffer("vec4", "v4f_fragData1", 1);
builder.addFragmentCode(glsl_perspectivePanelEmitOIT);
builder.setFragmentMain(`
v4f_fragData0 = vec4(0.0, 0.0, 0.0, 1.0);
v4f_fragData1 = vec4(0.0, 0.0, 0.0, 1.0);
vec4 v0 = getValue0();
gl_FragDepth = 1.0 - v0.r;
vec4 v0 = getValue0();
vec4 v1 = getValue1();
vec4 accum = vec4(v0.rgb, v1.r);
float revealage = v0.a;
emitAccumAndRevealage(accum, 1.0 - revealage, 0u);
`);
}

Expand Down Expand Up @@ -276,6 +276,7 @@ export class PerspectivePanel extends RenderedDataPanel {
protected visibleLayerTracker: Owned<
VisibleRenderLayerTracker<PerspectivePanel, PerspectiveViewRenderLayer>
>;
private hasVolumeRendering = false;

get rpc() {
return this.sharedObject.rpc!;
Expand All @@ -293,20 +294,18 @@ export class PerspectivePanel extends RenderedDataPanel {
// to avoid flickering when the camera is moving
private frameRateCalculator = new DownsamplingBasedOnFrameRateCalculator(
6 /* numberOfStoredFrameDeltas */,
4 /* maxDownsamplingFactor */,
8 /* maxDownsamplingFactor */,
8 /* desiredFrameTimingMs */,
60 /* downsamplingPersistenceDurationInFrames */,
);
private isCameraInContinuousMotion = false;
private isContinuousCameraMotionInProgress = false;
get shouldDownsample() {
return (
this.viewer.enableAdaptiveDownsampling.value &&
this.isCameraInContinuousMotion &&
this.isContinuousCameraMotionInProgress &&
this.hasVolumeRendering
);
}
private hasVolumeRendering = false;
private hasTransparent = false;

/**
* If boolean value is true, sliceView is shown unconditionally, regardless of the value of
Expand Down Expand Up @@ -374,12 +373,16 @@ export class PerspectivePanel extends RenderedDataPanel {
protected transparencyCopyHelper = this.registerDisposer(
OffscreenCopyHelper.get(this.gl, defineTransparencyCopyShader, 2),
);
protected transparentToTransparentCopyHelper = this.registerDisposer(
OffscreenCopyHelper.get(
this.gl,
defineTransparentToTransparentCopyShader,
2,
),
);
protected maxProjectionColorCopyHelper = this.registerDisposer(
OffscreenCopyHelper.get(this.gl, defineMaxProjectionColorCopyShader, 2),
);
protected offscreenDepthCopyHelper = this.registerDisposer(
OffscreenCopyHelper.get(this.gl, defineDepthCopyShader, 1),
);
protected maxProjectionPickCopyHelper = this.registerDisposer(
OffscreenCopyHelper.get(this.gl, defineMaxProjectionPickCopyShader, 2),
);
Expand Down Expand Up @@ -472,7 +475,7 @@ export class PerspectivePanel extends RenderedDataPanel {

this.registerDisposer(
this.context.continuousCameraMotionFinished.add(() => {
this.isCameraInContinuousMotion = false;
this.isContinuousCameraMotionInProgress = false;
if (this.hasVolumeRendering) {
this.scheduleRedraw();
this.frameRateCalculator.resetForNewFrameSet();
Expand All @@ -481,7 +484,7 @@ export class PerspectivePanel extends RenderedDataPanel {
);
this.registerDisposer(
this.context.continuousCameraMotionStarted.add(() => {
this.isCameraInContinuousMotion = true;
this.isContinuousCameraMotionInProgress = true;
}),
);

Expand Down Expand Up @@ -995,7 +998,7 @@ export class PerspectivePanel extends RenderedDataPanel {
frameNumber: this.context.frameNumber,
sliceViewsPresent: this.sliceViews.size > 0,
isContinuousCameraMotionInProgress:
this.context.isContinuousCameraMotionInProgress,
this.isContinuousCameraMotionInProgress,
};

mat4.copy(
Expand All @@ -1005,8 +1008,8 @@ export class PerspectivePanel extends RenderedDataPanel {

const { visibleLayers } = this.visibleLayerTracker;

this.hasTransparent = false;
let hasMaxProjection = false;
let hasTransparent = false;
let hasVolumeRenderingPick = false;
let hasAnnotation = false;
let hasVolumeRendering = false;

Expand All @@ -1019,11 +1022,14 @@ export class PerspectivePanel extends RenderedDataPanel {
hasAnnotation = true;
}
} else {
this.hasTransparent = true;
hasTransparent = true;
if (renderLayer.isVolumeRendering) {
hasVolumeRendering = true;
hasMaxProjection =
hasMaxProjection ||
// Volume rendering layers are not pickable when the camera is moving.
// Unless the layer is a projection layer.
hasVolumeRenderingPick =
hasVolumeRenderingPick ||
!this.isContinuousCameraMotionInProgress ||
isProjectionLayer(renderLayer as VolumeRenderingRenderLayer);
}
}
Expand Down Expand Up @@ -1070,7 +1076,7 @@ export class PerspectivePanel extends RenderedDataPanel {
/*dppass=*/ WebGL2RenderingContext.KEEP,
);

if (this.hasTransparent) {
if (hasTransparent) {
//Draw transparent objects.

let volumeRenderingBufferWidth = width;
Expand All @@ -1095,10 +1101,13 @@ export class PerspectivePanel extends RenderedDataPanel {
}
}

// Create max projection buffer if needed.
// Create volume rendering related buffers.
let bindMaxProjectionBuffer: () => void = () => {};
let bindMaxProjectionPickingBuffer: () => void = () => {};
if (hasMaxProjection) {
let bindVolumeRenderingBuffer: () => void = () => {};
if (this.hasVolumeRendering) {
// Max projection setup
renderContext.maxProjectionEmit = maxProjectionEmit;
const { maxProjectionConfiguration } = this;
bindMaxProjectionBuffer = () => {
maxProjectionConfiguration.bind(
Expand All @@ -1116,6 +1125,7 @@ export class PerspectivePanel extends RenderedDataPanel {
WebGL2RenderingContext.DEPTH_BUFFER_BIT,
);

// Max projection picking setup
const { maxProjectionPickConfiguration } = this;
bindMaxProjectionPickingBuffer = () => {
maxProjectionPickConfiguration.bind(
Expand All @@ -1128,28 +1138,22 @@ export class PerspectivePanel extends RenderedDataPanel {
WebGL2RenderingContext.COLOR_BUFFER_BIT |
WebGL2RenderingContext.DEPTH_BUFFER_BIT,
);
}

let bindVolumeRenderingBuffer: () => void = () => {};
if (hasVolumeRendering) {
// Volume rendering setup
bindVolumeRenderingBuffer = () => {
this.volumeRenderingConfiguration.bind(
volumeRenderingBufferWidth,
volumeRenderingBufferHeight,
);
};
bindVolumeRenderingBuffer();
// Copy the depth buffer from the offscreen framebuffer to the volume rendering framebuffer.
gl.depthMask(true);
renderContext.bindVolumeRenderingBuffer = bindVolumeRenderingBuffer;
gl.clearDepth(1.0);
gl.clearColor(0.0, 0.0, 0.0, 1.0);
gl.clear(
WebGL2RenderingContext.COLOR_BUFFER_BIT |
WebGL2RenderingContext.DEPTH_BUFFER_BIT,
);
this.offscreenDepthCopyHelper.draw(
this.offscreenFramebuffer.colorBuffers[OffscreenTextures.Z].texture,
);
}

const { transparentConfiguration } = this;
Expand All @@ -1174,30 +1178,58 @@ export class PerspectivePanel extends RenderedDataPanel {
let currentTransparentRenderingState =
TransparentRenderingState.TRANSPARENT;
for (const [renderLayer, attachment] of visibleLayers) {
if (renderLayer.isTransparent) {
if (renderLayer.isVolumeRendering) {
renderContext.depthBufferTexture =
this.offscreenFramebuffer.colorBuffers[OffscreenTextures.Z].texture;
}
// Draw max projection layers
if (
renderLayer.isVolumeRendering &&
isProjectionLayer(renderLayer as VolumeRenderingRenderLayer)
) {
// Set state for max projection mode and draw
gl.depthMask(true);
gl.disable(WebGL2RenderingContext.BLEND);
gl.depthFunc(WebGL2RenderingContext.GREATER);

if (
currentTransparentRenderingState !==
TransparentRenderingState.MAX_PROJECTION
) {
renderContext.emitter = maxProjectionEmit;
bindMaxProjectionBuffer();
const isVolumeProjectionLayer = isProjectionLayer(
renderLayer as VolumeRenderingRenderLayer,
);
const needsSecondPickingPass =
!isVolumeProjectionLayer &&
!this.isContinuousCameraMotionInProgress &&
!renderContext.wireFrame;

// Bind the appropriate buffer and set state
if (isVolumeProjectionLayer) {
gl.depthMask(true);
gl.disable(WebGL2RenderingContext.BLEND);
gl.depthFunc(WebGL2RenderingContext.GREATER);
if (
currentTransparentRenderingState !==
TransparentRenderingState.MAX_PROJECTION
) {
renderContext.emitter = maxProjectionEmit;
bindMaxProjectionBuffer();
}
} else {
if (
currentTransparentRenderingState !==
TransparentRenderingState.VOLUME_RENDERING
) {
renderContext.emitter = perspectivePanelEmitOIT;
bindVolumeRenderingBuffer();
}
gl.disable(WebGL2RenderingContext.DEPTH_TEST);
currentTransparentRenderingState =
TransparentRenderingState.VOLUME_RENDERING;
}

// Two cases for volume rendering layers
// Either way, a draw call is needed first
renderLayer.draw(renderContext, attachment);
gl.enable(WebGL2RenderingContext.DEPTH_TEST);

// Copy max projection result to picking buffer
// Case 1 - No picking pass needed and not a projection layer
// we already have the color information, so we skip the max projection pass
if (!needsSecondPickingPass && !isVolumeProjectionLayer) {
continue;
}

// Case 2 - Picking will be computed from a max projection
// And a second pass may be needed to do this picking

// Copy the volume rendering picking result to the main picking buffer
// Depth testing on to combine max layers into one pick buffer via depth
bindMaxProjectionPickingBuffer();
this.maxProjectionToPickCopyHelper.draw(
Expand All @@ -1207,25 +1239,30 @@ export class PerspectivePanel extends RenderedDataPanel {
this.maxProjectionConfiguration.colorBuffers[3 /*pick*/].texture,
);

// Copy max projection color result to the transparent buffer with OIT
// Depth testing off to combine max layers into one color via blend
bindVolumeRenderingBuffer();
gl.depthMask(false);
gl.disable(WebGL2RenderingContext.DEPTH_TEST);
// Turn back on OIT blending
gl.enable(WebGL2RenderingContext.BLEND);
gl.blendFuncSeparate(
WebGL2RenderingContext.ONE,
WebGL2RenderingContext.ONE,
WebGL2RenderingContext.ZERO,
WebGL2RenderingContext.ONE_MINUS_SRC_ALPHA,
);
this.maxProjectionColorCopyHelper.draw(
this.maxProjectionConfiguration.colorBuffers[0 /*color*/].texture,
this.maxProjectionConfiguration.colorBuffers[1 /*depth*/].texture,
);

// Reset the max projection buffer
// Copy max projection color result to the transparent buffer with OIT
// Depth testing off to combine max layers into one color via blending
if (isVolumeProjectionLayer) {
bindVolumeRenderingBuffer();
gl.depthMask(false);
gl.disable(WebGL2RenderingContext.DEPTH_TEST);
this.maxProjectionColorCopyHelper.draw(
this.maxProjectionConfiguration.colorBuffers[0 /*color*/].texture,
this.maxProjectionConfiguration.colorBuffers[1 /*depth*/].texture,
);
}

// Reset the max projection color, depth, and picking buffer
bindMaxProjectionBuffer();
renderContext.emitter = maxProjectionEmit;
gl.depthMask(true);
gl.clearColor(0.0, 0.0, 0.0, 0.0);
gl.clearDepth(0.0);
Expand All @@ -1234,7 +1271,7 @@ export class PerspectivePanel extends RenderedDataPanel {
WebGL2RenderingContext.DEPTH_BUFFER_BIT,
);

// Set back to non-max projection state
// Set some values back to non-max projection state
gl.clearDepth(1.0);
gl.clearColor(0.0, 0.0, 0.0, 1.0);
gl.depthMask(false);
Expand All @@ -1243,17 +1280,6 @@ export class PerspectivePanel extends RenderedDataPanel {

currentTransparentRenderingState =
TransparentRenderingState.MAX_PROJECTION;
} else if (renderLayer.isVolumeRendering) {
if (
currentTransparentRenderingState !==
TransparentRenderingState.VOLUME_RENDERING
) {
renderContext.emitter = perspectivePanelEmitOIT;
bindVolumeRenderingBuffer();
}
currentTransparentRenderingState =
TransparentRenderingState.VOLUME_RENDERING;
renderLayer.draw(renderContext, attachment);
}
// Draw regular transparent layers
else if (renderLayer.isTransparent) {
Expand All @@ -1271,18 +1297,18 @@ export class PerspectivePanel extends RenderedDataPanel {
}
// Copy transparent rendering result back to primary buffer.
gl.disable(WebGL2RenderingContext.DEPTH_TEST);
gl.viewport(0, 0, width, height);
this.offscreenFramebuffer.bindSingle(OffscreenTextures.COLOR);
gl.blendFunc(
WebGL2RenderingContext.ONE_MINUS_SRC_ALPHA,
WebGL2RenderingContext.SRC_ALPHA,
);
if (hasVolumeRendering) {
this.transparencyCopyHelper.draw(
renderContext.bindFramebuffer();
this.transparentToTransparentCopyHelper.draw(
this.volumeRenderingConfiguration.colorBuffers[0].texture,
this.volumeRenderingConfiguration.colorBuffers[1].texture,
);
}
gl.blendFunc(
WebGL2RenderingContext.ONE_MINUS_SRC_ALPHA,
WebGL2RenderingContext.SRC_ALPHA,
);
this.offscreenFramebuffer.bindSingle(OffscreenTextures.COLOR);
this.transparencyCopyHelper.draw(
transparentConfiguration.colorBuffers[0].texture,
transparentConfiguration.colorBuffers[1].texture,
Expand Down Expand Up @@ -1319,7 +1345,7 @@ export class PerspectivePanel extends RenderedDataPanel {
/*dppass=*/ WebGL2RenderingContext.REPLACE,
);
gl.stencilMask(2);
if (hasMaxProjection) {
if (hasVolumeRenderingPick) {
this.maxProjectionPickCopyHelper.draw(
this.maxProjectionPickConfiguration.colorBuffers[0].texture /*depth*/,
this.maxProjectionPickConfiguration.colorBuffers[1].texture /*pick*/,
Expand Down
Loading

0 comments on commit 4e378b4

Please sign in to comment.