Add a debanding property to Viewport for GLES3

It can be enabled in the Project Settings
(`rendering/quality/filters/use_debanding`). It's disabled
by default as it has a small performance impact and can make
PNG screenshots much larger (due to how dithering works).

As a result, it should be enabled only when banding is noticeable enough.

Since debanding requires a HDR viewport to work, it's only supported
in the GLES3 backend.

doc/classes/ProjectSettings.xml

@@ -1119,6 +1119,10 @@
 			Sets the number of MSAA samples to use. MSAA is used to reduce aliasing around the edges of polygons. A higher MSAA value results in smoother edges but can be significantly slower on some hardware.
 			[b]Note:[/b] MSAA is not available on HTML5 export using the GLES2 backend.
 		</member>
+		<member name="rendering/quality/filters/use_debanding" type="bool" setter="" getter="" default="false">
+			If [code]true[/code], uses a fast post-processing filter to make banding significantly less visible. In some cases, debanding may introduce a slightly noticeable dithering pattern. It's recommended to enable debanding only when actually needed since the dithering pattern will make lossless-compressed screenshots larger.
+			[b]Note:[/b] Only available on the GLES3 backend. [member rendering/quality/depth/hdr] must also be [code]true[/code] for debanding to be effective.
+		</member>
 		<member name="rendering/quality/filters/use_fxaa" type="bool" setter="" getter="" default="false">
 			Enables FXAA in the root Viewport. FXAA is a popular screen-space antialiasing method, which is fast but will make the image look blurry, especially at lower resolutions. It can still work relatively well at large resolutions such as 1440p and 4K.
 		</member>

doc/classes/Viewport.xml

@@ -235,6 +235,10 @@
 		<member name="canvas_transform" type="Transform2D" setter="set_canvas_transform" getter="get_canvas_transform">
 			The canvas transform of the viewport, useful for changing the on-screen positions of all child [CanvasItem]s. This is relative to the global canvas transform of the viewport.
 		</member>
+		<member name="debanding" type="bool" setter="set_use_debanding" getter="get_use_debanding" default="false">
+			If [code]true[/code], uses a fast post-processing filter to make banding significantly less visible. In some cases, debanding may introduce a slightly noticeable dithering pattern. It's recommended to enable debanding only when actually needed since the dithering pattern will make lossless-compressed screenshots larger.
+			[b]Note:[/b] Only available on the GLES3 backend. [member hdr] must also be [code]true[/code] for debanding to be effective.
+		</member>
 		<member name="debug_draw" type="int" setter="set_debug_draw" getter="get_debug_draw" enum="Viewport.DebugDraw" default="0">
 			The overlay mode for test rendered geometry in debug purposes.
 		</member>

doc/classes/VisualServer.xml

@@ -4231,6 +4231,18 @@
 				If [code]true[/code], the viewport uses augmented or virtual reality technologies. See [ARVRInterface].
 			</description>
 		</method>
+		<method name="viewport_set_use_debanding">
+			<return type="void">
+			</return>
+			<argument index="0" name="viewport" type="RID">
+			</argument>
+			<argument index="1" name="debanding" type="bool">
+			</argument>
+			<description>
+				If [code]true[/code], uses a fast post-processing filter to make banding significantly less visible. In some cases, debanding may introduce a slightly noticeable dithering pattern. It's recommended to enable debanding only when actually needed since the dithering pattern will make lossless-compressed screenshots larger.
+				[b]Note:[/b] Only available on the GLES3 backend. [member Viewport.hdr] must also be [code]true[/code] for debanding to be effective.
+			</description>
+		</method>
 		<method name="viewport_set_use_fxaa">
 			<return type="void">
 			</return>

drivers/dummy/rasterizer_dummy.h

@@ -709,6 +709,7 @@ public:
 	void render_target_clear_used(RID p_render_target) {}
 	void render_target_set_msaa(RID p_render_target, VS::ViewportMSAA p_msaa) {}
 	void render_target_set_use_fxaa(RID p_render_target, bool p_fxaa) {}
+	void render_target_set_use_debanding(RID p_render_target, bool p_debanding) {}
 
 	/* CANVAS SHADOW */
 

drivers/gles2/rasterizer_storage_gles2.cpp

@@ -5421,6 +5421,17 @@ void RasterizerStorageGLES2::render_target_set_use_fxaa(RID p_render_target, boo
 	rt->use_fxaa = p_fxaa;
 }
 
+void RasterizerStorageGLES2::render_target_set_use_debanding(RID p_render_target, bool p_debanding) {
+	RenderTarget *rt = render_target_owner.getornull(p_render_target);
+	ERR_FAIL_COND(!rt);
+
+	if (p_debanding) {
+		WARN_PRINT_ONCE("Debanding is not supported in the GLES2 backend. Switch to the GLES3 backend and make sure HDR is enabled.");
+	}
+
+	rt->use_debanding = p_debanding;
+}
+
 /* CANVAS SHADOW */
 
 RID RasterizerStorageGLES2::canvas_light_shadow_buffer_create(int p_width) {

drivers/gles2/rasterizer_storage_gles2.h

@@ -1219,6 +1219,7 @@ public:
 		VS::ViewportMSAA msaa;
 
 		bool use_fxaa;
+		bool use_debanding;
 
 		RID texture;
 
@@ -1240,6 +1241,7 @@ public:
 				used_in_frame(false),
 				msaa(VS::VIEWPORT_MSAA_DISABLED),
 				use_fxaa(false),
+				use_debanding(false),
 				used_dof_blur_near(false),
 				mip_maps_allocated(false) {
 			for (int i = 0; i < RENDER_TARGET_FLAG_MAX; ++i) {
@@ -1265,6 +1267,7 @@ public:
 	virtual void render_target_clear_used(RID p_render_target);
 	virtual void render_target_set_msaa(RID p_render_target, VS::ViewportMSAA p_msaa);
 	virtual void render_target_set_use_fxaa(RID p_render_target, bool p_fxaa);
+	virtual void render_target_set_use_debanding(RID p_render_target, bool p_debanding);
 
 	/* CANVAS SHADOW */
 

drivers/gles3/rasterizer_scene_gles3.cpp

@@ -3627,7 +3627,7 @@ void RasterizerSceneGLES3::_post_process(Environment *env, const CameraMatrix &p
 		glBindFramebuffer(GL_DRAW_FRAMEBUFFER, 0);
 	}
 
-	if ((!env || storage->frame.current_rt->flags[RasterizerStorage::RENDER_TARGET_TRANSPARENT] || storage->frame.current_rt->width < 4 || storage->frame.current_rt->height < 4) && !storage->frame.current_rt->use_fxaa) { //no post process on small render targets
+	if ((!env || storage->frame.current_rt->flags[RasterizerStorage::RENDER_TARGET_TRANSPARENT] || storage->frame.current_rt->width < 4 || storage->frame.current_rt->height < 4) && !storage->frame.current_rt->use_fxaa && !storage->frame.current_rt->use_debanding) { //no post process on small render targets
 		//no environment or transparent render, simply return and convert to SRGB
 		if (storage->frame.current_rt->external.fbo != 0) {
 			glBindFramebuffer(GL_FRAMEBUFFER, storage->frame.current_rt->external.fbo);
@@ -3994,6 +3994,7 @@ void RasterizerSceneGLES3::_post_process(Environment *env, const CameraMatrix &p
 
 	state.tonemap_shader.set_conditional(TonemapShaderGLES3::KEEP_3D_LINEAR, storage->frame.current_rt->flags[RasterizerStorage::RENDER_TARGET_KEEP_3D_LINEAR]);
 	state.tonemap_shader.set_conditional(TonemapShaderGLES3::USE_FXAA, storage->frame.current_rt->use_fxaa);
+	state.tonemap_shader.set_conditional(TonemapShaderGLES3::USE_DEBANDING, storage->frame.current_rt->use_debanding);
 
 	if (env && max_glow_level >= 0) {
 
@@ -4083,6 +4084,7 @@ void RasterizerSceneGLES3::_post_process(Environment *env, const CameraMatrix &p
 
 	//turn off everything used
 	state.tonemap_shader.set_conditional(TonemapShaderGLES3::USE_FXAA, false);
+	state.tonemap_shader.set_conditional(TonemapShaderGLES3::USE_DEBANDING, false);
 	state.tonemap_shader.set_conditional(TonemapShaderGLES3::USE_AUTO_EXPOSURE, false);
 	state.tonemap_shader.set_conditional(TonemapShaderGLES3::USE_FILMIC_TONEMAPPER, false);
 	state.tonemap_shader.set_conditional(TonemapShaderGLES3::USE_ACES_TONEMAPPER, false);

drivers/gles3/rasterizer_storage_gles3.cpp

@@ -7761,6 +7761,14 @@ void RasterizerStorageGLES3::render_target_set_use_fxaa(RID p_render_target, boo
 	rt->use_fxaa = p_fxaa;
 }
 
+void RasterizerStorageGLES3::render_target_set_use_debanding(RID p_render_target, bool p_debanding) {
+
+	RenderTarget *rt = render_target_owner.getornull(p_render_target);
+	ERR_FAIL_COND(!rt);
+
+	rt->use_debanding = p_debanding;
+}
+
 /* CANVAS SHADOW */
 
 RID RasterizerStorageGLES3::canvas_light_shadow_buffer_create(int p_width) {

drivers/gles3/rasterizer_storage_gles3.h

@@ -1391,6 +1391,7 @@ public:
 		bool used_in_frame;
 		VS::ViewportMSAA msaa;
 		bool use_fxaa;
+		bool use_debanding;
 
 		RID texture;
 
@@ -1402,7 +1403,8 @@ public:
 				height(0),
 				used_in_frame(false),
 				msaa(VS::VIEWPORT_MSAA_DISABLED),
-				use_fxaa(false) {
+				use_fxaa(false),
+				use_debanding(false) {
 			exposure.fbo = 0;
 			buffers.fbo = 0;
 			external.fbo = 0;
@@ -1431,6 +1433,7 @@ public:
 	virtual void render_target_clear_used(RID p_render_target);
 	virtual void render_target_set_msaa(RID p_render_target, VS::ViewportMSAA p_msaa);
 	virtual void render_target_set_use_fxaa(RID p_render_target, bool p_fxaa);
+	virtual void render_target_set_use_debanding(RID p_render_target, bool p_debanding);
 
 	/* CANVAS SHADOW */
 

drivers/gles3/shaders/tonemap.glsl

@@ -312,6 +312,18 @@ vec3 apply_fxaa(vec3 color, float exposure, vec2 uv_interp, vec2 pixel_size) {
 	}
 }
 
+// From http://alex.vlachos.com/graphics/Alex_Vlachos_Advanced_VR_Rendering_GDC2015.pdf
+// and https://www.shadertoy.com/view/MslGR8 (5th one starting from the bottom)
+// NOTE: `frag_coord` is in pixels (i.e. not normalized UV).
+vec3 screen_space_dither(vec2 frag_coord) {
+	// Iestyn's RGB dither (7 asm instructions) from Portal 2 X360, slightly modified for VR.
+	vec3 dither = vec3(dot(vec2(171.0, 231.0), frag_coord));
+	dither.rgb = fract(dither.rgb / vec3(103.0, 71.0, 97.0));
+
+	// Subtract 0.5 to avoid slightly brightening the whole viewport.
+	return (dither.rgb - 0.5) / 255.0;
+}
+
 void main() {
 	vec3 color = textureLod(source, uv_interp, 0.0f).rgb;
 
@@ -328,6 +340,12 @@ void main() {
 	color = apply_fxaa(color, exposure, uv_interp, pixel_size);
 #endif
 
+#ifdef USE_DEBANDING
+	// For best results, debanding should be done before tonemapping.
+	// Otherwise, we're adding noise to an already-quantized image.
+	color += screen_space_dither(gl_FragCoord.xy);
+#endif
+
 	// Early Tonemap & SRGB Conversion; note that Linear tonemapping does not clamp to [0, 1]; some operations below expect a [0, 1] range and will clamp
 
 	color = apply_tonemapping(color, white);

editor/plugins/spatial_editor_plugin.cpp

@@ -2478,7 +2478,7 @@ void SpatialEditorViewport::_notification(int p_what) {
 			viewport_container->set_stretch_shrink(shrink ? 2 : 1);
 		}
 
-		//update msaa if changed
+		// Update MSAA, FXAA, debanding and HDR if changed.
 
 		int msaa_mode = ProjectSettings::get_singleton()->get("rendering/quality/filters/msaa");
 		viewport->set_msaa(Viewport::MSAA(msaa_mode));
@@ -2486,6 +2486,9 @@ void SpatialEditorViewport::_notification(int p_what) {
 		bool use_fxaa = ProjectSettings::get_singleton()->get("rendering/quality/filters/use_fxaa");
 		viewport->set_use_fxaa(use_fxaa);
 
+		bool use_debanding = ProjectSettings::get_singleton()->get("rendering/quality/filters/use_debanding");
+		viewport->set_use_debanding(use_debanding);
+
 		bool hdr = ProjectSettings::get_singleton()->get("rendering/quality/depth/hdr");
 		viewport->set_hdr(hdr);
 

scene/main/scene_tree.cpp

@@ -2082,6 +2082,9 @@ SceneTree::SceneTree() {
 	const bool use_fxaa = GLOBAL_DEF("rendering/quality/filters/use_fxaa", false);
 	root->set_use_fxaa(use_fxaa);
 
+	const bool use_debanding = GLOBAL_DEF("rendering/quality/filters/use_debanding", false);
+	root->set_use_debanding(use_debanding);
+
 	GLOBAL_DEF_RST("rendering/quality/depth/hdr", true);
 	GLOBAL_DEF("rendering/quality/depth/hdr.mobile", false);
 

scene/main/viewport.cpp

@@ -3029,6 +3029,17 @@ bool Viewport::get_use_fxaa() const {
 	return use_fxaa;
 }
 
+void Viewport::set_use_debanding(bool p_debanding) {
+
+	use_debanding = p_debanding;
+	VS::get_singleton()->viewport_set_use_debanding(viewport, use_debanding);
+}
+
+bool Viewport::get_use_debanding() const {
+
+	return use_debanding;
+}
+
 void Viewport::set_hdr(bool p_hdr) {
 
 	if (hdr == p_hdr)
@@ -3167,6 +3178,9 @@ void Viewport::_bind_methods() {
 	ClassDB::bind_method(D_METHOD("set_use_fxaa", "enable"), &Viewport::set_use_fxaa);
 	ClassDB::bind_method(D_METHOD("get_use_fxaa"), &Viewport::get_use_fxaa);
 
+	ClassDB::bind_method(D_METHOD("set_use_debanding", "enable"), &Viewport::set_use_debanding);
+	ClassDB::bind_method(D_METHOD("get_use_debanding"), &Viewport::get_use_debanding);
+
 	ClassDB::bind_method(D_METHOD("set_hdr", "enable"), &Viewport::set_hdr);
 	ClassDB::bind_method(D_METHOD("get_hdr"), &Viewport::get_hdr);
 
@@ -3256,6 +3270,7 @@ void Viewport::_bind_methods() {
 	ADD_GROUP("Rendering", "");
 	ADD_PROPERTY(PropertyInfo(Variant::INT, "msaa", PROPERTY_HINT_ENUM, "Disabled,2x,4x,8x,16x,AndroidVR 2x,AndroidVR 4x"), "set_msaa", "get_msaa");
 	ADD_PROPERTY(PropertyInfo(Variant::BOOL, "fxaa"), "set_use_fxaa", "get_use_fxaa");
+	ADD_PROPERTY(PropertyInfo(Variant::BOOL, "debanding"), "set_use_debanding", "get_use_debanding");
 	ADD_PROPERTY(PropertyInfo(Variant::BOOL, "hdr"), "set_hdr", "get_hdr");
 	ADD_PROPERTY(PropertyInfo(Variant::BOOL, "disable_3d"), "set_disable_3d", "is_3d_disabled");
 	ADD_PROPERTY(PropertyInfo(Variant::BOOL, "keep_3d_linear"), "set_keep_3d_linear", "get_keep_3d_linear");
@@ -3417,6 +3432,7 @@ Viewport::Viewport() {
 
 	msaa = MSAA_DISABLED;
 	use_fxaa = false;
+	use_debanding = false;
 	hdr = true;
 
 	usage = USAGE_3D;

scene/main/viewport.h

@@ -283,6 +283,7 @@ private:
 
 	MSAA msaa;
 	bool use_fxaa;
+	bool use_debanding;
 	bool hdr;
 
 	Ref<ViewportTexture> default_texture;
@@ -499,6 +500,9 @@ public:
 	void set_use_fxaa(bool p_fxaa);
 	bool get_use_fxaa() const;
 
+	void set_use_debanding(bool p_debanding);
+	bool get_use_debanding() const;
+
 	void set_hdr(bool p_hdr);
 	bool get_hdr() const;
 

servers/visual/rasterizer.h

@@ -573,6 +573,7 @@ public:
 	virtual void render_target_clear_used(RID p_render_target) = 0;
 	virtual void render_target_set_msaa(RID p_render_target, VS::ViewportMSAA p_msaa) = 0;
 	virtual void render_target_set_use_fxaa(RID p_render_target, bool p_fxaa) = 0;
+	virtual void render_target_set_use_debanding(RID p_render_target, bool p_debanding) = 0;
 
 	/* CANVAS SHADOW */
 

servers/visual/visual_server_raster.h

@@ -486,6 +486,7 @@ public:
 	BIND3(viewport_set_shadow_atlas_quadrant_subdivision, RID, int, int)
 	BIND2(viewport_set_msaa, RID, ViewportMSAA)
 	BIND2(viewport_set_use_fxaa, RID, bool)
+	BIND2(viewport_set_use_debanding, RID, bool)
 	BIND2(viewport_set_hdr, RID, bool)
 	BIND2(viewport_set_usage, RID, ViewportUsage)
 

servers/visual/visual_server_viewport.cpp

@@ -661,6 +661,14 @@ void VisualServerViewport::viewport_set_use_fxaa(RID p_viewport, bool p_fxaa) {
 	VSG::storage->render_target_set_use_fxaa(viewport->render_target, p_fxaa);
 }
 
+void VisualServerViewport::viewport_set_use_debanding(RID p_viewport, bool p_debanding) {
+
+	Viewport *viewport = viewport_owner.getornull(p_viewport);
+	ERR_FAIL_COND(!viewport);
+
+	VSG::storage->render_target_set_use_debanding(viewport->render_target, p_debanding);
+}
+
 void VisualServerViewport::viewport_set_hdr(RID p_viewport, bool p_enabled) {
 
 	Viewport *viewport = viewport_owner.getornull(p_viewport);

servers/visual/visual_server_viewport.h

@@ -191,6 +191,7 @@ public:
 
 	void viewport_set_msaa(RID p_viewport, VS::ViewportMSAA p_msaa);
 	void viewport_set_use_fxaa(RID p_viewport, bool p_fxaa);
+	void viewport_set_use_debanding(RID p_viewport, bool p_debanding);
 	void viewport_set_hdr(RID p_viewport, bool p_enabled);
 	void viewport_set_usage(RID p_viewport, VS::ViewportUsage p_usage);
 

servers/visual/visual_server_wrap_mt.h

@@ -413,6 +413,7 @@ public:
 	FUNC3(viewport_set_shadow_atlas_quadrant_subdivision, RID, int, int)
 	FUNC2(viewport_set_msaa, RID, ViewportMSAA)
 	FUNC2(viewport_set_use_fxaa, RID, bool)
+	FUNC2(viewport_set_use_debanding, RID, bool)
 	FUNC2(viewport_set_hdr, RID, bool)
 	FUNC2(viewport_set_usage, RID, ViewportUsage)
 

servers/visual_server.cpp

@@ -1892,6 +1892,7 @@ void VisualServer::_bind_methods() {
 	ClassDB::bind_method(D_METHOD("viewport_set_shadow_atlas_quadrant_subdivision", "viewport", "quadrant", "subdivision"), &VisualServer::viewport_set_shadow_atlas_quadrant_subdivision);
 	ClassDB::bind_method(D_METHOD("viewport_set_msaa", "viewport", "msaa"), &VisualServer::viewport_set_msaa);
 	ClassDB::bind_method(D_METHOD("viewport_set_use_fxaa", "viewport", "fxaa"), &VisualServer::viewport_set_use_fxaa);
+	ClassDB::bind_method(D_METHOD("viewport_set_use_debanding", "viewport", "debanding"), &VisualServer::viewport_set_use_debanding);
 	ClassDB::bind_method(D_METHOD("viewport_set_hdr", "viewport", "enabled"), &VisualServer::viewport_set_hdr);
 	ClassDB::bind_method(D_METHOD("viewport_set_usage", "viewport", "usage"), &VisualServer::viewport_set_usage);
 	ClassDB::bind_method(D_METHOD("viewport_get_render_info", "viewport", "info"), &VisualServer::viewport_get_render_info);

servers/visual_server.h

@@ -681,6 +681,7 @@ public:
 
 	virtual void viewport_set_msaa(RID p_viewport, ViewportMSAA p_msaa) = 0;
 	virtual void viewport_set_use_fxaa(RID p_viewport, bool p_fxaa) = 0;
+	virtual void viewport_set_use_debanding(RID p_viewport, bool p_debanding) = 0;
 
 	enum ViewportUsage {
 		VIEWPORT_USAGE_2D,