Merge branch 'pkristof/sanitizePomNans' into 'main'

[REMIX-3709]: Sanitize nans in POM uvs

See merge request lightspeedrtx/dxvk-remix-nv!1152

Author: pkristof

src/dxvk/shaders/rtx/concept/surface_material/opaque_surface_material_interaction.slangh

@@ -87,7 +87,15 @@ float4 pomGetPatchCorners(Texture2D texture, SamplerState sampler, float2 boxCen
 }
 
 // cast a ray in tangent space.  return an intersection point in tangent point (or an exit point if z > 1.0f)
-float3 pomTraceRay(Texture2D texture, SamplerState sampler, float3 origin, float3 direction, float2 textureGradientX, float2 textureGradientY, inout uint iterations, float neutralHeight)
+float3 pomTraceRay(
+  Texture2D texture,
+  SamplerState sampler,
+  float3 origin,
+  float3 direction,
+  float2 textureGradientX,
+  float2 textureGradientY,
+  inout uint iterations,
+  float neutralHeight)
 {
   // Potential future improvements for outwards rays:
   // 1) calculate starting height the same way the box intersection code does instead of just sampling at a pixel.
@@ -155,7 +163,7 @@ float3 pomTraceRay(Texture2D texture, SamplerState sampler, float3 origin, float
         // causes smooth slops to turn jagged. It may be an improvement to use that approach only when the current
         // pixel is the largest or smallest of all the neighbors, but that seemed too complex to be worth pursuing.
 
-        // Calculate the point where  the ray entered this box:
+        // Calculate the point where the ray entered this box:
         const float2 boxNearCorner = boxCenter - forwardHalfStep * boxSize;
         const float2 boxEntranceDist = (boxNearCorner - origin.xy) / direction.xy;
         const float3 boxEntrance = origin + max(0.0f, max(boxEntranceDist.x, boxEntranceDist.y)) * direction;
@@ -217,10 +225,12 @@ float3 pomTraceRay(Texture2D texture, SamplerState sampler, float3 origin, float
   }
 
   // Outwards rays can go past the top of the POM surface, so limit it to there.
-  if (curPos.z > 1) {
+  if (curPos.z > 1)
+  {
     float distToOne = (1.0f - origin.z) / direction.z;
     return origin + distToOne * direction;
   }
+
   return curPos;
 }
 
@@ -445,13 +455,17 @@ OpaqueSurfaceMaterialInteraction opaqueSurfaceMaterialInteractionCreate(
     f16vec3 viewDirTextureSpace = normalize(mul(worldToTexture, minimalRayInteraction.viewDirection));
     // Note: -1.f * TextureGradientX has the same x,y direction as viewDirTextureSpace, but calculating the correct z value is non-obvious.
     // Could potentially remove the need for the worldToTexture transform if a faster way to calculate that z value is found.
-
+    
     uint iterations = 0; // unused
-    surfaceInteraction.textureCoordinates = pomCalculateTexcoord(
+    const vec2 pomTextureCoordinates = pomCalculateTexcoord(
         opaqueSurfaceMaterial,
         surfaceInteraction,
         viewDirTextureSpace,
         iterations).xy;
+
+    // WAR for REMIX-3709 - POM sometimes produces NaNs.
+    surfaceInteraction.textureCoordinates = sanitize(pomTextureCoordinates, surfaceInteraction.textureCoordinates);
+
     opaqueSurfaceMaterialInteraction.flags |= OPAQUE_SURFACE_MATERIAL_INTERACTION_FLAG_HAS_HEIGHT_TEXTURE;
   }
 #endif

src/dxvk/shaders/rtx/utility/common.slangh

@@ -82,15 +82,23 @@ bool isValidValue(vec4 value)
   return !(any(isnan(value)) || any(isinf(value)));
 }
 
-float sanitize(float val, float defaultVal) {
+float sanitize(float val, float defaultVal)
+{
   return isValidValue(val) ? val : defaultVal;
 }
 
-vec3 sanitize(vec3 val, vec3 defaultVal) {
+vec2 sanitize(vec2 val, vec2 defaultVal)
+{
   return isValidValue(val) ? val : defaultVal;
 }
 
-vec4 sanitize(vec4 val, vec4 defaultVal) {
+vec3 sanitize(vec3 val, vec3 defaultVal)
+{
+  return isValidValue(val) ? val : defaultVal;
+}
+
+vec4 sanitize(vec4 val, vec4 defaultVal)
+{
   return isValidValue(val) ? val : defaultVal;
 }
 

src/dxvk/shaders/rtx/utility/math.slangh

@@ -322,7 +322,7 @@ f16vec4 quaternionMultiply(f16vec4 q1, f16vec4 q2, bool signBitIsParity = false)
 // Calculates a tangent and bitangent vector in the direction of the U and V coordinates of a triangle orthonormal to a
 // specified normal (this may be a triangle normal or an interpolated normal depending on the use case).
 // Note that the vectors this function returns may form an improper rotation when converted to a transformation due to a swap of
-// handedness. No gaurantees for left or right handedness are made as it depends on the triangle's UV layout.
+// handedness. No guarantees for left or right handedness are made as it depends on the triangle's UV layout.
 void genTangSpace(vec2 uvs[3],
                   vec3 pos[3],
                   f16vec3 normal,
@@ -358,7 +358,7 @@ void genTangSpace(vec2 uvs[3],
     rawTangent = (duv12[1] * dp02 - duv02[1] * dp12) * invdet;
     rawBitangent = (-duv12[0] * dp02 + duv02[0] * dp12) * invdet;
   }
-  // rawTangent and rawBitangent are too large for fp16, but We need the non-uniform component of the scale for POM.
+  // rawTangent and rawBitangent are too large for fp16, but we need the non-uniform component of the scale for POM.
   // Here, we scale both down by the length of the larger vector.
   float largerRawLength = sqrt(max(squareLength(rawTangent), squareLength(rawBitangent)));