Fix and improve FXAA

src/engine/renderer/gl_shader.cpp

@@ -795,6 +795,21 @@ static std::string GenEngineConstants() {
 		AddDefine( str, "r_colorGrading", 1 );
 	}
 
+	if ( r_showLuminance.Get() )
+	{
+		AddDefine( str, "r_showLuminance", 1 );
+	}
+
+	if ( r_FXAA.Get() )
+	{
+		AddDefine( str, "r_FXAA", 1 );
+
+		if ( r_showFXAA.Get() )
+		{
+			AddDefine( str, "r_showFXAA", 1 );
+		}
+	}
+
 	if ( r_highPrecisionRendering.Get() ) {
 		AddDefine( str, "r_highPrecisionRendering", 1 );
 	}
@@ -3066,4 +3081,4 @@ GlobalUBOProxy::GlobalUBOProxy() :
 	u_Tonemap( this ),
 	u_TonemapParms( this ),
 	u_Exposure( this ) {
-}
+}

src/engine/renderer/glsl_source/cameraEffects_fp.glsl

@@ -111,5 +111,23 @@ void main()
 
 	color.xyz = pow(color.xyz, vec3(u_InverseGamma));
 
+	#if defined(r_FXAA) || defined(r_showLuminance)
+	{
+		// That luminance vector comes from a comment in fxaa3_11_fp.glsl.
+		vec3 luminanceVector = vec3( 0.299, 0.587, 0.114 );
+
+		float luminance = dot( color.rgb, luminanceVector );
+
+		#if defined(r_showLuminance)
+			color.rgb = vec3( luminance );
+		#endif
+
+		#if defined(r_FXAA)
+			// Encode luminance in alpha channel.
+			color.a = luminance;
+		#endif
+	}
+	#endif
+
 	outputColor = color;
 }

src/engine/renderer/glsl_source/fxaa3_11_fp.glsl

@@ -20,20 +20,6 @@ DAMAGES.
 
 kangz: This code has been set in the public domain by TIMOTHY LOTTES
 
-============================================================================*/
-
-//Due to our shader system, we put the defines for the control knobs here
-#define FXAA_PC 1
-#if __VERSION__ == 120
-#define FXAA_GLSL_120 1
-#else
-#define FXAA_GLSL_130 1
-#endif
-
-#define FXAA_QUALITY_PRESET 12
-#define FXAA_GREEN_AS_LUMA 1
-
-/*============================================================================
 ------------------------------------------------------------------------------
                            INTEGRATION CHECKLIST
 ------------------------------------------------------------------------------

src/engine/renderer/glsl_source/fxaa_fp.glsl

@@ -33,13 +33,25 @@ ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 
 /* fxaa_fp.glsl */
 
-// The FXAA parameters are put directly in fxaa3_11_fp.glsl
-// because we cannot #include in the middle of a shader
-// ^This is no longer true, but I'm not touching that mess
+// Control knobs.
+#if __VERSION__ == 120
+#define FXAA_GLSL_120 1
+#else
+#define FXAA_GLSL_130 1
+#endif
+
+#define FXAA_PC 1
+#define FXAA_QUALITY_PRESET 12
+#define FXAA_GREEN_AS_LUMA 0
 
 #insert fxaa3_11_fp
 
+#if defined(HAVE_ARB_bindless_texture)
+uniform sampler2D u_ColorMap_linear;
+#define u_ColorMap u_ColorMap_linear
+#else
 uniform sampler2D	u_ColorMap;
+#endif
 
 #if __VERSION__ > 120
 out vec4 outputColor;
@@ -49,7 +61,7 @@ out vec4 outputColor;
 
 void	main()
 {
-	outputColor = FxaaPixelShader(
+	vec4 color = FxaaPixelShader(
 		gl_FragCoord.xy / r_FBufSize, //pos
 		vec4(0.0), //not used
 		u_ColorMap, //tex
@@ -67,4 +79,19 @@ void	main()
 		0.0, //not used
 		vec4(0.0) //not used
 	);
+
+	#if defined(r_showFXAA)
+	{
+		vec4 originalColor = FxaaTexTop( u_ColorMap, gl_FragCoord.xy / r_FBufSize );
+
+		if ( color.r != originalColor.r
+			|| color.g != originalColor.g
+			|| color.b != originalColor.b )
+		{
+			color.rgb = vec3(1.0, 0.0, 0.0);
+		}
+	}
+	#endif
+
+	outputColor = vec4( color.rgb, 1.0f );
 }

src/engine/renderer/tr_backend.cpp

@@ -1603,7 +1603,7 @@ void RB_RenderSSAO()
 
 void RB_FXAA()
 {
-	if ( !r_FXAA->integer || !gl_fxaaShader )
+	if ( !r_FXAA.Get() || !gl_fxaaShader )
 	{
 		return;
 	}
@@ -1621,15 +1621,71 @@ void RB_FXAA()
 	// set the shader parameters
 	gl_fxaaShader->BindProgram();
 
-	// Swap main FBOs
 	gl_fxaaShader->SetUniform_ColorMapBindless(
 		GL_BindToTMU( 0, tr.currentRenderImage[backEnd.currentMainFBO] )
 	);
-	backEnd.currentMainFBO = 1 - backEnd.currentMainFBO;
-	R_BindFBO( tr.mainFBO[ backEnd.currentMainFBO ] );
+
+	bool greaterThanGL33 = std::make_pair( glConfig.glMajor, glConfig.glMinor ) >= std::make_pair( 3, 3 );
+
+	GLuint64 handle = 0;
+
+	// FXAA expect GL_LINEAR for the sampling to work.
+	if ( greaterThanGL33 )
+	{
+		GLuint sampler = 0;
+		glGenSamplers( 1, &sampler );
+		glSamplerParameteri( sampler, GL_TEXTURE_MAG_FILTER, GL_LINEAR) ;
+		glSamplerParameteri( sampler, GL_TEXTURE_MIN_FILTER, GL_LINEAR );
+
+		if ( glConfig.usingBindlessTextures )
+		{
+			// Set a handler.
+			GLuint texture = tr.currentRenderImage[backEnd.currentMainFBO]->texnum;
+			handle = glGetTextureSamplerHandleARB( texture, sampler );
+			glMakeTextureHandleResidentARB( handle );
+			GLuint program = gl_fxaaShader->GetProgram()->id;
+			GLint location = glGetUniformLocation( program, "u_ColorMap_linear" );
+			glUniformHandleui64ARB( location, handle );
+		}
+		else
+		{
+			// Bind a sampler.
+			glBindSampler( 0, sampler );
+		}
+	}
+	else
+	{
+		// Change the filter (requires a texture bind).
+		glTexParameterf( GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR );
+		glTexParameterf( GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR );
+	}
+
+	// This shader is run last, so let it render to screen.
+	R_BindNullFBO();
 
 	Tess_InstantScreenSpaceQuad();
 
+	// Make sure we didn't break other effects expecting GL_NEAREST.
+	if ( greaterThanGL33 )
+	{
+		if ( glConfig.usingBindlessTextures )
+		{
+			// Unset the handler.
+			glMakeTextureHandleNonResidentARB( handle );
+		}
+		else
+		{
+			// Unbind the sampler.
+			glBindSampler( 0, 0 );
+		}
+	}
+	else
+	{
+		// Restore the filter (requires a texture bind).
+		glTexParameterf( GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST );
+		glTexParameterf( GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST );
+	}
+
 	GL_CheckErrors();
 }
 
@@ -1693,13 +1749,22 @@ void RB_CameraPostFX() {
 	}
 	gl_cameraEffectsShader->SetUniform_Tonemap( tonemap );
 
-	// This shader is run last, so let it render to screen instead of
-	// tr.mainFBO
-	R_BindNullFBO();
 	gl_cameraEffectsShader->SetUniform_CurrentMapBindless(
 		GL_BindToTMU( 0, tr.currentRenderImage[backEnd.currentMainFBO] ) 
 	);
 
+	if ( r_FXAA.Get() && gl_fxaaShader )
+	{
+		// Swap main FBOs.
+		backEnd.currentMainFBO = 1 - backEnd.currentMainFBO;
+		R_BindFBO( tr.mainFBO[ backEnd.currentMainFBO ] );
+	}
+	else
+	{
+		// Without FXAA this shader is run last, so let it render to screen.
+		R_BindNullFBO();
+	}
+
 	if ( glConfig.colorGrading ) {
 		gl_cameraEffectsShader->SetUniform_ColorMap3DBindless( GL_BindToTMU( 3, tr.colorGradeImage ) );
 	}
@@ -2810,11 +2875,11 @@ static void RB_RenderPostProcess()
 
 	TransitionMSAAToMain( GL_COLOR_BUFFER_BIT );
 
-	RB_FXAA();
-
 	// render chromatic aberration
 	RB_CameraPostFX();
 
+	RB_FXAA();
+
 	// copy to given byte buffer that is NOT a FBO
 	if ( tr.refdef.pixelTarget != nullptr ) {
 		glReadPixels( 0, 0, tr.refdef.pixelTargetWidth, tr.refdef.pixelTargetHeight, GL_RGBA,

src/engine/renderer/tr_init.cpp

@@ -285,8 +285,14 @@ Cvar::Cvar<int> r_rendererAPI( "r_rendererAPI", "Renderer API: 0: OpenGL, 1: Vul
 	Cvar::Cvar<bool> r_bloom( "r_bloom", "Use bloom", Cvar::ARCHIVE, false );
 	Cvar::Cvar<float> r_bloomBlur( "r_bloomBlur", "Bloom strength", Cvar::NONE, 0.2 );
 	Cvar::Cvar<int> r_bloomPasses( "r_bloomPasses", "Amount of bloom passes in each direction", Cvar::NONE, 2 );
-	cvar_t      *r_FXAA;
+
+	Cvar::Cvar<bool> r_showLuminance( "r_showLuminance", "Show luminance", Cvar::CHEAT, false );
+
+	Cvar::Cvar<bool> r_FXAA( "r_FXAA", "Fast approximate anti-aliasing", Cvar::NONE, false );
+	Cvar::Cvar<bool> r_showFXAA( "r_showFXAA", "Show pixels modified by FXAA", Cvar::CHEAT, false );
+
 	Cvar::Range<Cvar::Cvar<int>> r_msaa( "r_msaa", "Amount of MSAA samples. 0 to disable", Cvar::NONE, 0, 0, 64 );
+
 	Cvar::Range<Cvar::Cvar<int>> r_ssao( "r_ssao",
 		"Screen space ambient occlusion: "
 		"-1: show, 0: disabled, 1: enabled",
@@ -1205,8 +1211,13 @@ ScreenshotCmd screenshotPNGRegistration("screenshotPNG", ssFormat_t::SSF_PNG, "p
 		r_printShaders = Cvar_Get( "r_printShaders", "0", 0 );
 
 		Cvar::Latch( r_bloom );
-		r_FXAA = Cvar_Get( "r_FXAA", "0", CVAR_LATCH | CVAR_ARCHIVE );
 		Cvar::Latch( r_ssao );
+
+		Cvar::Latch( r_showLuminance );
+
+		Cvar::Latch( r_FXAA );
+		Cvar::Latch( r_showFXAA );
+
 		Cvar::Latch( r_msaa );
 
 		// temporary variables that can change at any time

src/engine/renderer/tr_local.h

@@ -2778,8 +2778,13 @@ enum
 	extern Cvar::Cvar<bool> r_bloom;
 	extern Cvar::Cvar<float> r_bloomBlur;
 	extern Cvar::Cvar<int> r_bloomPasses;
-	extern cvar_t *r_FXAA;
 	extern Cvar::Range<Cvar::Cvar<int>> r_ssao;
+
+	extern Cvar::Cvar<bool> r_showLuminance;
+
+	extern Cvar::Cvar<bool> r_FXAA;
+	extern Cvar::Cvar<bool> r_showFXAA;
+
 	extern Cvar::Range<Cvar::Cvar<int>> r_msaa;
 
 	extern cvar_t *r_evsmPostProcess;

src/engine/renderer/tr_shade.cpp

@@ -374,7 +374,7 @@ static void GLSL_InitGPUShadersOrError()
 		gl_ssaoShader->MarkProgramForBuilding();
 	}
 
-	if ( r_FXAA->integer != 0 )
+	if ( r_FXAA.Get() )
 	{
 		gl_shaderManager.LoadShader( gl_fxaaShader );