EnvironmentMapLight, BRDF Improvements (#7051)

(Before)
![image](https://user-images.githubusercontent.com/47158642/213946111-15ec758f-1f1d-443c-b196-1fdcd4ae49da.png)
(After)
![image](https://user-images.githubusercontent.com/47158642/217051179-67381e73-dd44-461b-a2c7-87b0440ef8de.png)
![image](https://user-images.githubusercontent.com/47158642/212492404-524e4ad3-7837-4ed4-8b20-2abc276aa8e8.png)

# Objective
- Improve lighting; especially reflections.
- Closes #4581.

## Solution
- Implement environment maps, providing better ambient light.
- Add microfacet multibounce approximation for specular highlights from Filament.
- Occlusion is no longer incorrectly applied to direct lighting. It now only applies to diffuse indirect light. Unsure if it's also supposed to apply to specular indirect light - the glTF specification just says "indirect light". In the case of ambient occlusion, for instance, that's usually only calculated as diffuse though. For now, I'm choosing to apply this just to indirect diffuse light, and not specular.
- Modified the PBR example to use an environment map, and have labels.
- Added `FallbackImageCubemap`.

## Implementation
- IBL technique references can be found in environment_map.wgsl.
- It's more accurate to use a LUT for the scale/bias. Filament has a good reference on generating this LUT. For now, I just used an analytic approximation.
 - For now, environment maps must first be prefiltered outside of bevy using a 3rd party tool. See the `EnvironmentMap` documentation.
- Eventually, we should have our own prefiltering code, so that we can have dynamically changing environment maps, as well as let users drop in an HDR image and use asset preprocessing to create the needed textures using only bevy. 

---

## Changelog
- Added an `EnvironmentMapLight` camera component that adds additional ambient light to a scene.
- StandardMaterials will now appear brighter and more saturated at high roughness, due to internal material changes. This is more physically correct.
- Fixed StandardMaterial occlusion being incorrectly applied to direct lighting.
- Added `FallbackImageCubemap`.

Co-authored-by: IceSentry <[email protected]>
Co-authored-by: James Liu <[email protected]>
Co-authored-by: Rob Parrett <[email protected]>

Author: 4 people

.github/workflows/ci.yml

@@ -190,14 +190,14 @@ jobs:
       - name: Build bevy
         # this uses the same command as when running the example to ensure build is reused
         run: |
-          TRACE_CHROME=trace-alien_cake_addict.json CI_TESTING_CONFIG=.github/example-run/alien_cake_addict.ron cargo build --example alien_cake_addict --features "bevy_ci_testing,trace,trace_chrome"
+          TRACE_CHROME=trace-alien_cake_addict.json CI_TESTING_CONFIG=.github/example-run/alien_cake_addict.ron cargo build --example alien_cake_addict --features "bevy_ci_testing,trace,trace_chrome,ktx2,zstd"
       - name: Run examples
         run: |
           for example in .github/example-run/*.ron; do
             example_name=`basename $example .ron`
             echo -n $example_name > last_example_run
             echo "running $example_name - "`date`
-            time TRACE_CHROME=trace-$example_name.json CI_TESTING_CONFIG=$example xvfb-run cargo run --example $example_name --features "bevy_ci_testing,trace,trace_chrome"
+            time TRACE_CHROME=trace-$example_name.json CI_TESTING_CONFIG=$example xvfb-run cargo run --example $example_name --features "bevy_ci_testing,trace,trace_chrome,ktx2,zstd"
             sleep 10
           done
           zip traces.zip trace*.json

.github/workflows/validation-jobs.yml

@@ -80,15 +80,15 @@ jobs:
         shell: bash
         # this uses the same command as when running the example to ensure build is reused
         run: |
-          WGPU_BACKEND=dx12 CI_TESTING_CONFIG=.github/example-run/alien_cake_addict.ron cargo build --example alien_cake_addict --features "bevy_ci_testing"
+          WGPU_BACKEND=dx12 CI_TESTING_CONFIG=.github/example-run/alien_cake_addict.ron cargo build --example alien_cake_addict --features "bevy_ci_testing,ktx2,zstd"
 
       - name: Run examples
         shell: bash
         run: |
           for example in .github/example-run/*.ron; do
             example_name=`basename $example .ron`
             echo "running $example_name - "`date`
-            time WGPU_BACKEND=dx12 CI_TESTING_CONFIG=$example cargo run --example $example_name --features "bevy_ci_testing"
+            time WGPU_BACKEND=dx12 CI_TESTING_CONFIG=$example cargo run --example $example_name --features "bevy_ci_testing,ktx2,zstd"
             sleep 10
           done
 

Cargo.toml

@@ -372,6 +372,7 @@ wasm = false
 [[example]]
 name = "load_gltf"
 path = "examples/3d/load_gltf.rs"
+required-features = ["ktx2", "zstd"]
 
 [package.metadata.example.load_gltf]
 name = "Load glTF"
@@ -422,6 +423,7 @@ wasm = true
 [[example]]
 name = "pbr"
 path = "examples/3d/pbr.rs"
+required-features = ["ktx2", "zstd"]
 
 [package.metadata.example.pbr]
 name = "Physically Based Rendering"
@@ -1430,6 +1432,7 @@ wasm = true
 [[example]]
 name = "scene_viewer"
 path = "examples/tools/scene_viewer/main.rs"
+required-features = ["ktx2", "zstd"]
 
 [package.metadata.example.scene_viewer]
 name = "Scene Viewer"

assets/environment_maps/info.txt

@@ -0,0 +1,6 @@
+The pisa_*.ktx2 files were generated from https://github.com/KhronosGroup/glTF-Sample-Environments/blob/master/pisa.hdr using the following tools and commands:
+- IBL environment map prefiltering to cubemaps: https://github.com/KhronosGroup/glTF-IBL-Sampler
+  - Diffuse: ./cli -inputPath pisa.hdr -outCubeMap pisa_diffuse.ktx2 -distribution Lambertian -cubeMapResolution 32
+  - Specular: ./cli -inputPath pisa.hdr -outCubeMap pisa_specular.ktx2 -distribution GGX -cubeMapResolution 512
+- Converting to rgb9e5 format with zstd 'supercompression': https://github.com/DGriffin91/bevy_mod_environment_map_tools
+  - cargo run --release -- --inputs pisa_diffuse.ktx2,pisa_specular.ktx2 --outputs pisa_diffuse_rgb9e5_zstd.ktx2,pisa_specular_rgb9e5_zstd.ktx2

assets/environment_maps/pisa_diffuse_rgb9e5_zstd.ktx2

@@ -0,0 +1,43 @@
+#define_import_path bevy_pbr::environment_map
+
+
+struct EnvironmentMapLight {
+    diffuse: vec3<f32>,
+    specular: vec3<f32>,
+};
+
+fn environment_map_light(
+    perceptual_roughness: f32,
+    roughness: f32,
+    diffuse_color: vec3<f32>,
+    NdotV: f32,
+    f_ab: vec2<f32>,
+    N: vec3<f32>,
+    R: vec3<f32>,
+    F0: vec3<f32>,
+) -> EnvironmentMapLight {
+
+    // Split-sum approximation for image based lighting: https://cdn2.unrealengine.com/Resources/files/2013SiggraphPresentationsNotes-26915738.pdf
+    let smallest_specular_mip_level = textureNumLevels(environment_map_specular) - 1i;
+    let radiance_level = perceptual_roughness * f32(smallest_specular_mip_level);
+    let irradiance = textureSample(environment_map_diffuse, environment_map_sampler, N).rgb;
+    let radiance = textureSampleLevel(environment_map_specular, environment_map_sampler, R, radiance_level).rgb;
+
+    // Multiscattering approximation: https://www.jcgt.org/published/0008/01/03/paper.pdf
+    // Useful reference: https://bruop.github.io/ibl
+    let Fr = max(vec3(1.0 - roughness), F0) - F0;
+    let kS = F0 + Fr * pow(1.0 - NdotV, 5.0);
+    let FssEss = kS * f_ab.x + f_ab.y;
+    let Ess = f_ab.x + f_ab.y;
+    let Ems = 1.0 - Ess;
+    let Favg = F0 + (1.0 - F0) / 21.0;
+    let Fms = FssEss * Favg / (1.0 - Ems * Favg);
+    let FmsEms = Fms * Ems;
+    let Edss = 1.0 - (FssEss + FmsEms);
+    let kD = diffuse_color * Edss;
+
+    var out: EnvironmentMapLight;
+    out.diffuse = (FmsEms + kD) * irradiance;
+    out.specular = FssEss * radiance;
+    return out;
+}

assets/environment_maps/pisa_specular_rgb9e5_zstd.ktx2

@@ -0,0 +1,137 @@
+use bevy_app::{App, Plugin};
+use bevy_asset::{load_internal_asset, Handle, HandleUntyped};
+use bevy_core_pipeline::prelude::Camera3d;
+use bevy_ecs::{prelude::Component, query::With};
+use bevy_reflect::{Reflect, TypeUuid};
+use bevy_render::{
+    extract_component::{ExtractComponent, ExtractComponentPlugin},
+    render_asset::RenderAssets,
+    render_resource::{
+        BindGroupEntry, BindGroupLayoutEntry, BindingResource, BindingType, SamplerBindingType,
+        Shader, ShaderStages, TextureSampleType, TextureViewDimension,
+    },
+    texture::{FallbackImageCubemap, Image},
+};
+
+pub const ENVIRONMENT_MAP_SHADER_HANDLE: HandleUntyped =
+    HandleUntyped::weak_from_u64(Shader::TYPE_UUID, 154476556247605696);
+
+pub struct EnvironmentMapPlugin;
+
+impl Plugin for EnvironmentMapPlugin {
+    fn build(&self, app: &mut App) {
+        load_internal_asset!(
+            app,
+            ENVIRONMENT_MAP_SHADER_HANDLE,
+            "environment_map.wgsl",
+            Shader::from_wgsl
+        );
+
+        app.register_type::<EnvironmentMapLight>()
+            .add_plugin(ExtractComponentPlugin::<EnvironmentMapLight>::default());
+    }
+}
+
+/// Environment map based ambient lighting representing light from distant scenery.
+///
+/// When added to a 3D camera, this component adds indirect light
+/// to every point of the scene (including inside, enclosed areas) based on
+/// an environment cubemap texture. This is similiar to [`crate::AmbientLight`], but
+/// higher quality, and is intended for outdoor scenes.
+///
+/// The environment map must be prefiltered into a diffuse and specular cubemap based on the
+/// [split-sum approximation](https://cdn2.unrealengine.com/Resources/files/2013SiggraphPresentationsNotes-26915738.pdf).
+///
+/// To prefilter your environment map, you can use `KhronosGroup`'s [glTF-IBL-Sampler](https://github.com/KhronosGroup/glTF-IBL-Sampler).
+/// The diffuse map uses the Lambertian distribution, and the specular map uses the GGX distribution.
+///
+/// `KhronosGroup` also has several prefiltered environment maps that can be found [here](https://github.com/KhronosGroup/glTF-Sample-Environments).
+#[derive(Component, Reflect, Clone)]
+pub struct EnvironmentMapLight {
+    pub diffuse_map: Handle<Image>,
+    pub specular_map: Handle<Image>,
+}
+
+impl EnvironmentMapLight {
+    /// Whether or not all textures neccesary to use the environment map
+    /// have been loaded by the asset server.
+    pub fn is_loaded(&self, images: &RenderAssets<Image>) -> bool {
+        images.get(&self.diffuse_map).is_some() && images.get(&self.specular_map).is_some()
+    }
+}
+
+impl ExtractComponent for EnvironmentMapLight {
+    type Query = &'static Self;
+    type Filter = With<Camera3d>;
+    type Out = Self;
+
+    fn extract_component(item: bevy_ecs::query::QueryItem<'_, Self::Query>) -> Option<Self::Out> {
+        Some(item.clone())
+    }
+}
+
+pub fn get_bindings<'a>(
+    environment_map_light: Option<&EnvironmentMapLight>,
+    images: &'a RenderAssets<Image>,
+    fallback_image_cubemap: &'a FallbackImageCubemap,
+    bindings: [u32; 3],
+) -> [BindGroupEntry<'a>; 3] {
+    let (diffuse_map, specular_map) = match (
+        environment_map_light.and_then(|env_map| images.get(&env_map.diffuse_map)),
+        environment_map_light.and_then(|env_map| images.get(&env_map.specular_map)),
+    ) {
+        (Some(diffuse_map), Some(specular_map)) => {
+            (&diffuse_map.texture_view, &specular_map.texture_view)
+        }
+        _ => (
+            &fallback_image_cubemap.texture_view,
+            &fallback_image_cubemap.texture_view,
+        ),
+    };
+
+    [
+        BindGroupEntry {
+            binding: bindings[0],
+            resource: BindingResource::TextureView(diffuse_map),
+        },
+        BindGroupEntry {
+            binding: bindings[1],
+            resource: BindingResource::TextureView(specular_map),
+        },
+        BindGroupEntry {
+            binding: bindings[2],
+            resource: BindingResource::Sampler(&fallback_image_cubemap.sampler),
+        },
+    ]
+}
+
+pub fn get_bind_group_layout_entries(bindings: [u32; 3]) -> [BindGroupLayoutEntry; 3] {
+    [
+        BindGroupLayoutEntry {
+            binding: bindings[0],
+            visibility: ShaderStages::FRAGMENT,
+            ty: BindingType::Texture {
+                sample_type: TextureSampleType::Float { filterable: true },
+                view_dimension: TextureViewDimension::Cube,
+                multisampled: false,
+            },
+            count: None,
+        },
+        BindGroupLayoutEntry {
+            binding: bindings[1],
+            visibility: ShaderStages::FRAGMENT,
+            ty: BindingType::Texture {
+                sample_type: TextureSampleType::Float { filterable: true },
+                view_dimension: TextureViewDimension::Cube,
+                multisampled: false,
+            },
+            count: None,
+        },
+        BindGroupLayoutEntry {
+            binding: bindings[2],
+            visibility: ShaderStages::FRAGMENT,
+            ty: BindingType::Sampler(SamplerBindingType::Filtering),
+            count: None,
+        },
+    ]
+}

crates/bevy_pbr/src/environment_map/environment_map.wgsl

@@ -2,6 +2,7 @@ pub mod wireframe;
 
 mod alpha;
 mod bundle;
+mod environment_map;
 mod fog;
 mod light;
 mod material;
@@ -10,8 +11,8 @@ mod prepass;
 mod render;
 
 pub use alpha::*;
-use bevy_transform::TransformSystem;
 pub use bundle::*;
+pub use environment_map::EnvironmentMapLight;
 pub use fog::*;
 pub use light::*;
 pub use material::*;
@@ -27,6 +28,7 @@ pub mod prelude {
             DirectionalLightBundle, MaterialMeshBundle, PbrBundle, PointLightBundle,
             SpotLightBundle,
         },
+        environment_map::EnvironmentMapLight,
         fog::{FogFalloff, FogSettings},
         light::{AmbientLight, DirectionalLight, PointLight, SpotLight},
         material::{Material, MaterialPlugin},
@@ -55,6 +57,8 @@ use bevy_render::{
     view::{ViewSet, VisibilitySystems},
     ExtractSchedule, RenderApp, RenderSet,
 };
+use bevy_transform::TransformSystem;
+use environment_map::EnvironmentMapPlugin;
 
 pub const PBR_TYPES_SHADER_HANDLE: HandleUntyped =
     HandleUntyped::weak_from_u64(Shader::TYPE_UUID, 1708015359337029744);
@@ -172,6 +176,7 @@ impl Plugin for PbrPlugin {
                 prepass_enabled: self.prepass_enabled,
                 ..Default::default()
             })
+            .add_plugin(EnvironmentMapPlugin)
             .init_resource::<AmbientLight>()
             .init_resource::<GlobalVisiblePointLights>()
             .init_resource::<DirectionalLightShadowMap>()

crates/bevy_pbr/src/environment_map/mod.rs

@@ -1,6 +1,6 @@
 use crate::{
-    AlphaMode, DrawMesh, MeshPipeline, MeshPipelineKey, MeshUniform, PrepassPlugin,
-    SetMeshBindGroup, SetMeshViewBindGroup,
+    AlphaMode, DrawMesh, EnvironmentMapLight, MeshPipeline, MeshPipelineKey, MeshUniform,
+    PrepassPlugin, SetMeshBindGroup, SetMeshViewBindGroup,
 };
 use bevy_app::{App, Plugin};
 use bevy_asset::{AddAsset, AssetEvent, AssetServer, Assets, Handle};
@@ -361,10 +361,12 @@ pub fn queue_material_meshes<M: Material>(
     render_meshes: Res<RenderAssets<Mesh>>,
     render_materials: Res<RenderMaterials<M>>,
     material_meshes: Query<(&Handle<M>, &Handle<Mesh>, &MeshUniform)>,
+    images: Res<RenderAssets<Image>>,
     mut views: Query<(
         &ExtractedView,
         &VisibleEntities,
         Option<&Tonemapping>,
+        Option<&EnvironmentMapLight>,
         &mut RenderPhase<Opaque3d>,
         &mut RenderPhase<AlphaMask3d>,
         &mut RenderPhase<Transparent3d>,
@@ -376,6 +378,7 @@ pub fn queue_material_meshes<M: Material>(
         view,
         visible_entities,
         tonemapping,
+        environment_map,
         mut opaque_phase,
         mut alpha_mask_phase,
         mut transparent_phase,
@@ -388,6 +391,14 @@ pub fn queue_material_meshes<M: Material>(
         let mut view_key = MeshPipelineKey::from_msaa_samples(msaa.samples())
             | MeshPipelineKey::from_hdr(view.hdr);
 
+        let environment_map_loaded = match environment_map {
+            Some(environment_map) => environment_map.is_loaded(&images),
+            None => false,
+        };
+        if environment_map_loaded {
+            view_key |= MeshPipelineKey::ENVIRONMENT_MAP;
+        }
+
         if let Some(Tonemapping::Enabled { deband_dither }) = tonemapping {
             if !view.hdr {
                 view_key |= MeshPipelineKey::TONEMAP_IN_SHADER;
@@ -397,8 +408,8 @@ pub fn queue_material_meshes<M: Material>(
                 }
             }
         }
-        let rangefinder = view.rangefinder3d();
 
+        let rangefinder = view.rangefinder3d();
         for visible_entity in &visible_entities.entities {
             if let Ok((material_handle, mesh_handle, mesh_uniform)) =
                 material_meshes.get(*visible_entity)