Try to limit darkening caused by bad deluxe maps

src/engine/renderer/glsl_source/computeLight_fp.glsl

@@ -72,7 +72,7 @@ vec4 EnvironmentalSpecularFactor( vec3 viewDir, vec3 normal )
 
 #if defined(USE_DELUXE_MAPPING) || defined(USE_GRID_DELUXE_MAPPING) || defined(r_realtimeLighting)
 void computeDeluxeLight( vec3 lightDir, vec3 normal, vec3 viewDir, vec3 lightColor,
-	vec4 diffuseColor, vec4 materialColor,
+	vec3 lambertTerm, vec4 diffuseColor, vec4 materialColor,
 	inout vec4 color )
 {
 	vec3 H = normalize( lightDir + viewDir );
@@ -92,6 +92,10 @@ void computeDeluxeLight( vec3 lightDir, vec3 normal, vec3 viewDir, vec3 lightCol
 
 	NdotL = clamp( NdotL, 0.0, 1.0 );
 
+	#if !defined(USE_LIGHT_MAPPING)
+		lambertTerm = NdotL * lightColor;
+	#endif
+
 	#if defined(USE_PHYSICAL_MAPPING)
 		// Daemon PBR packing defaults to ORM like glTF 2.0 defines
 		// https://www.khronos.org/blog/art-pipeline-for-gltf
@@ -124,11 +128,11 @@ void computeDeluxeLight( vec3 lightDir, vec3 normal, vec3 viewDir, vec3 lightCol
 
 		vec3 diffuseBRDF = NdotL * diffuseColor.rgb * ( 1.0 - metalness );
 		vec3 specularBRDF = vec3( ( D * F * G ) / max( 4.0 * NdotL * NdotV, 0.0001f ) );
-		color.rgb += ( diffuseBRDF + specularBRDF ) * lightColor.rgb * NdotL;
+		color.rgb += ( diffuseBRDF + specularBRDF ) * lambertTerm;
 		color.a = mix( diffuseColor.a, 1.0, FexpNV );
 
 	#else // !USE_PHYSICAL_MAPPING
-		color.rgb += lightColor.rgb * NdotL * diffuseColor.rgb;
+		color.rgb += lambertTerm * diffuseColor.rgb;
 		#if defined(r_specularMapping)
 			color.rgb += computeSpecularity(lightColor.rgb, materialColor, NdotH);
 		#endif // r_specularMapping
@@ -198,8 +202,9 @@ void computeDynamicLight( uint idx, vec3 P, vec3 normal, vec3 viewDir, vec4 diff
 		attenuation = 1.0;
 	}
 
+	vec3 attenuatedColor = attenuation * attenuation * light.color;
 	computeDeluxeLight(
-		L, normal, viewDir, attenuation * attenuation * light.color,
+		L, normal, viewDir, attenuatedColor, dot(L, normal) * attenuatedColor,
 		diffuse, material, color );
 }
 

src/engine/renderer/glsl_source/lightMapping_fp.glsl

@@ -163,6 +163,8 @@ void main()
 	#endif
 
 	#if defined(USE_LIGHT_MAPPING) && defined(USE_DELUXE_MAPPING)
+		vec3 lightmapValue = lightColor;
+
 		/* Lightmaps generated by q3map2 don't store the raw light value, but
 		they store light premultiplied with the dot product of the light
 		direction and surface normal. The line is just an attempt to reverse
@@ -183,18 +185,39 @@ void main()
 		Increasing the value should reduce these artifacts. -- gimhael
 		https://github.com/DaemonEngine/Daemon/issues/299#issuecomment-606186347
 		*/
-
-		// Divide by cosine term to restore original light color.
-		lightColor /= clamp(dot(normalize(var_Normal), lightDir), 0.3, 1.0);
+		vec3 flatNormal = normalize(var_Normal);
+		float flatNdotL = clamp(dot(flatNormal, lightDir), 0.3, 1.0);
+		lightColor /= flatNdotL;
+
+		// The light direction from the deluxe map may be nonsense, be it because q3map2
+		// goofed, or because the light actually comes from more than one direction(!!!!). The total, summed
+		// over light sources, of (light color) * dot(flat triangle's normal, light direction), as given by the lightmap, is more
+		// trustworthy. If the normalmapped normal is almost the same as the the original flat geometry's normal,
+		// we can bound how much the above sum can change, thus preventing the bogus light direction from
+		// wrecking the Lambert's law term of our lighting equation.
+		// So we want to compute the maximum of
+		//     abs(dot(var_Normal, L) - dot(normal, L))
+		//   = abs(dot(normal - var_Normal, L))
+		// over all possible L. This maximum occurs when L is parallel to normal - var_Normal.
+		float dotDelta = length(normal - flatNormal);
+		// So each dot product term can only increase or decrease by dotDelta. But we don't know
+		// what the original dot products were. We'll have to assume a maximum angle to be able to
+		// bound the percent change of the total.
+		float minimumAverageCosine = 0.1; // 84.3 degrees
+		float cosinesRatio = dot(normal, lightDir) / flatNdotL;
+		cosinesRatio = clamp( cosinesRatio, 1.0 - dotDelta / minimumAverageCosine, 1.0 + dotDelta / minimumAverageCosine );
+		vec3 lambertTerm = lightmapValue * cosinesRatio;
+	#else
+		vec3 lambertTerm; // not used
 	#endif
 
 	// Blend static light.
 	#if defined(USE_DELUXE_MAPPING) || defined(USE_GRID_DELUXE_MAPPING)
 		#if defined(USE_REFLECTIVE_SPECULAR)
 			vec4 modifiedSpecular = material * EnvironmentalSpecularFactor(viewDir, normal);
-			computeDeluxeLight(lightDir, normal, viewDir, lightColor, diffuse, modifiedSpecular, color);
+			computeDeluxeLight(lightDir, normal, viewDir, lightColor, lambertTerm, diffuse, modifiedSpecular, color);
 		#else // !USE_REFLECTIVE_SPECULAR
-			computeDeluxeLight(lightDir, normal, viewDir, lightColor, diffuse, material, color);
+			computeDeluxeLight(lightDir, normal, viewDir, lightColor, lambertTerm, diffuse, material, color);
 		#endif // !USE_REFLECTIVE_SPECULAR
 	#else
 		computeLight(lightColor, diffuse, color);