shader_recompiler: Add workaround for drivers with unexpected unorm rounding behavior. (#2310)

Author: squidbus

src/shader_recompiler/frontend/translate/export.cpp

@@ -17,7 +17,11 @@ u32 SwizzleMrtComponent(const FragmentRuntimeInfo::PsColorBuffer& color_buffer,
 
 void Translator::ExportMrtValue(IR::Attribute attribute, u32 comp, const IR::F32& value,
                                 const FragmentRuntimeInfo::PsColorBuffer& color_buffer) {
-    const auto converted = ApplyWriteNumberConversion(ir, value, color_buffer.num_conversion);
+    auto converted = ApplyWriteNumberConversion(ir, value, color_buffer.num_conversion);
+    if (color_buffer.needs_unorm_fixup) {
+        // FIXME: Fix-up for GPUs where float-to-unorm rounding is off from expected.
+        converted = ir.FPSub(converted, ir.Imm32(1.f / 127500.f));
+    }
     ir.SetAttribute(attribute, converted, comp);
 }
 

src/shader_recompiler/runtime_info.h

@@ -185,6 +185,7 @@ struct FragmentRuntimeInfo {
         AmdGpu::NumberConversion num_conversion;
         AmdGpu::CompMapping swizzle;
         AmdGpu::Liverpool::ShaderExportFormat export_format;
+        bool needs_unorm_fixup;
 
         auto operator<=>(const PsColorBuffer&) const noexcept = default;
     };

src/video_core/renderer_vulkan/vk_pipeline_cache.cpp

@@ -330,13 +330,24 @@ bool PipelineCache::RefreshGraphicsKey() {
             continue;
         }
 
+        // Metal seems to have an issue where 8-bit unorm/snorm/sRGB outputs to render target
+        // need a bias applied to round correctly; detect and set the flag for that here.
+        const auto needs_unorm_fixup = instance.GetDriverID() == vk::DriverId::eMoltenvk &&
+                                       (col_buf.GetNumberFmt() == AmdGpu::NumberFormat::Unorm ||
+                                        col_buf.GetNumberFmt() == AmdGpu::NumberFormat::Snorm ||
+                                        col_buf.GetNumberFmt() == AmdGpu::NumberFormat::Srgb) &&
+                                       (col_buf.GetDataFmt() == AmdGpu::DataFormat::Format8 ||
+                                        col_buf.GetDataFmt() == AmdGpu::DataFormat::Format8_8 ||
+                                        col_buf.GetDataFmt() == AmdGpu::DataFormat::Format8_8_8_8);
+
         key.color_formats[remapped_cb] =
             LiverpoolToVK::SurfaceFormat(col_buf.GetDataFmt(), col_buf.GetNumberFmt());
         key.color_buffers[remapped_cb] = {
             .num_format = col_buf.GetNumberFmt(),
             .num_conversion = col_buf.GetNumberConversion(),
             .swizzle = col_buf.Swizzle(),
             .export_format = regs.color_export_format.GetFormat(cb),
+            .needs_unorm_fixup = needs_unorm_fixup,
         };
     }