use std::num::NonZeroU64;
use bevy_ecs::{
component::Component,
entity::{Entity, EntityHashMap},
query::Without,
reflect::ReflectComponent,
system::{Commands, Query, Res, Resource},
world::{FromWorld, World},
};
use bevy_math::{AspectRatio, UVec2, UVec3, UVec4, Vec3Swizzles as _, Vec4};
use bevy_reflect::{std_traits::ReflectDefault, Reflect};
use bevy_render::{
camera::Camera,
render_resource::{
BindingResource, BufferBindingType, ShaderSize as _, ShaderType, StorageBuffer,
UniformBuffer,
},
renderer::{RenderDevice, RenderQueue},
Extract,
};
use bevy_utils::{hashbrown::HashSet, tracing::warn};
pub(crate) use crate::cluster::assign::assign_objects_to_clusters;
use crate::MeshPipeline;
mod assign;
#[cfg(test)]
mod test;
pub const MAX_UNIFORM_BUFFER_CLUSTERABLE_OBJECTS: usize = 256;
pub const CLUSTERED_FORWARD_STORAGE_BUFFER_COUNT: u32 = 3;
const CLUSTER_COUNT_SIZE: u32 = 9;
const CLUSTER_OFFSET_MASK: u32 = (1 << (32 - (CLUSTER_COUNT_SIZE * 2))) - 1;
const CLUSTER_COUNT_MASK: u32 = (1 << CLUSTER_COUNT_SIZE) - 1;
#[derive(Debug, Copy, Clone, Reflect)]
pub enum ClusterFarZMode {
MaxClusterableObjectRange,
Constant(f32),
}
#[derive(Debug, Copy, Clone, Reflect)]
#[reflect(Default)]
pub struct ClusterZConfig {
pub first_slice_depth: f32,
pub far_z_mode: ClusterFarZMode,
}
#[derive(Debug, Copy, Clone, Component, Reflect)]
#[reflect(Component)]
pub enum ClusterConfig {
None,
Single,
XYZ {
dimensions: UVec3,
z_config: ClusterZConfig,
dynamic_resizing: bool,
},
FixedZ {
total: u32,
z_slices: u32,
z_config: ClusterZConfig,
dynamic_resizing: bool,
},
}
#[derive(Component, Debug, Default)]
pub struct Clusters {
pub(crate) tile_size: UVec2,
pub(crate) dimensions: UVec3,
pub(crate) near: f32,
pub(crate) far: f32,
pub(crate) clusterable_objects: Vec<VisibleClusterableObjects>,
}
#[derive(Clone, Component, Debug, Default)]
pub struct VisibleClusterableObjects {
pub(crate) entities: Vec<Entity>,
pub point_light_count: usize,
pub spot_light_count: usize,
}
#[derive(Resource, Default)]
pub struct GlobalVisibleClusterableObjects {
pub(crate) entities: HashSet<Entity>,
}
#[derive(Resource)]
pub struct GlobalClusterableObjectMeta {
pub gpu_clusterable_objects: GpuClusterableObjects,
pub entity_to_index: EntityHashMap<usize>,
}
#[derive(Copy, Clone, ShaderType, Default, Debug)]
pub struct GpuClusterableObject {
pub(crate) light_custom_data: Vec4,
pub(crate) color_inverse_square_range: Vec4,
pub(crate) position_radius: Vec4,
pub(crate) flags: u32,
pub(crate) shadow_depth_bias: f32,
pub(crate) shadow_normal_bias: f32,
pub(crate) spot_light_tan_angle: f32,
}
pub enum GpuClusterableObjects {
Uniform(UniformBuffer<GpuClusterableObjectsUniform>),
Storage(StorageBuffer<GpuClusterableObjectsStorage>),
}
#[derive(ShaderType)]
pub struct GpuClusterableObjectsUniform {
data: Box<[GpuClusterableObject; MAX_UNIFORM_BUFFER_CLUSTERABLE_OBJECTS]>,
}
#[derive(ShaderType, Default)]
pub struct GpuClusterableObjectsStorage {
#[size(runtime)]
data: Vec<GpuClusterableObject>,
}
#[derive(Component)]
pub struct ExtractedClusterConfig {
pub(crate) near: f32,
pub(crate) far: f32,
pub(crate) dimensions: UVec3,
}
enum ExtractedClusterableObjectElement {
ClusterHeader(u32, u32),
ClusterableObjectEntity(Entity),
}
#[derive(Component)]
pub struct ExtractedClusterableObjects {
data: Vec<ExtractedClusterableObjectElement>,
}
#[derive(ShaderType)]
struct GpuClusterOffsetsAndCountsUniform {
data: Box<[UVec4; ViewClusterBindings::MAX_UNIFORM_ITEMS]>,
}
#[derive(ShaderType, Default)]
struct GpuClusterableObjectIndexListsStorage {
#[size(runtime)]
data: Vec<u32>,
}
#[derive(ShaderType, Default)]
struct GpuClusterOffsetsAndCountsStorage {
#[size(runtime)]
data: Vec<UVec4>,
}
enum ViewClusterBuffers {
Uniform {
clusterable_object_index_lists: UniformBuffer<GpuClusterableObjectIndexListsUniform>,
cluster_offsets_and_counts: UniformBuffer<GpuClusterOffsetsAndCountsUniform>,
},
Storage {
clusterable_object_index_lists: StorageBuffer<GpuClusterableObjectIndexListsStorage>,
cluster_offsets_and_counts: StorageBuffer<GpuClusterOffsetsAndCountsStorage>,
},
}
#[derive(Component)]
pub struct ViewClusterBindings {
n_indices: usize,
n_offsets: usize,
buffers: ViewClusterBuffers,
}
impl Default for ClusterZConfig {
fn default() -> Self {
Self {
first_slice_depth: 5.0,
far_z_mode: ClusterFarZMode::MaxClusterableObjectRange,
}
}
}
impl Default for ClusterConfig {
fn default() -> Self {
Self::FixedZ {
total: 4096,
z_slices: 24,
z_config: ClusterZConfig::default(),
dynamic_resizing: true,
}
}
}
impl ClusterConfig {
fn dimensions_for_screen_size(&self, screen_size: UVec2) -> UVec3 {
match &self {
ClusterConfig::None => UVec3::ZERO,
ClusterConfig::Single => UVec3::ONE,
ClusterConfig::XYZ { dimensions, .. } => *dimensions,
ClusterConfig::FixedZ {
total, z_slices, ..
} => {
let aspect_ratio: f32 =
AspectRatio::from_pixels(screen_size.x, screen_size.y).into();
let mut z_slices = *z_slices;
if *total < z_slices {
warn!("ClusterConfig has more z-slices than total clusters!");
z_slices = *total;
}
let per_layer = *total as f32 / z_slices as f32;
let y = f32::sqrt(per_layer / aspect_ratio);
let mut x = (y * aspect_ratio) as u32;
let mut y = y as u32;
if x == 0 {
x = 1;
y = per_layer as u32;
}
if y == 0 {
x = per_layer as u32;
y = 1;
}
UVec3::new(x, y, z_slices)
}
}
}
fn first_slice_depth(&self) -> f32 {
match self {
ClusterConfig::None | ClusterConfig::Single => 0.0,
ClusterConfig::XYZ { z_config, .. } | ClusterConfig::FixedZ { z_config, .. } => {
z_config.first_slice_depth
}
}
}
fn far_z_mode(&self) -> ClusterFarZMode {
match self {
ClusterConfig::None => ClusterFarZMode::Constant(0.0),
ClusterConfig::Single => ClusterFarZMode::MaxClusterableObjectRange,
ClusterConfig::XYZ { z_config, .. } | ClusterConfig::FixedZ { z_config, .. } => {
z_config.far_z_mode
}
}
}
fn dynamic_resizing(&self) -> bool {
match self {
ClusterConfig::None | ClusterConfig::Single => false,
ClusterConfig::XYZ {
dynamic_resizing, ..
}
| ClusterConfig::FixedZ {
dynamic_resizing, ..
} => *dynamic_resizing,
}
}
}
impl Clusters {
fn update(&mut self, screen_size: UVec2, requested_dimensions: UVec3) {
debug_assert!(
requested_dimensions.x > 0 && requested_dimensions.y > 0 && requested_dimensions.z > 0
);
let tile_size = (screen_size.as_vec2() / requested_dimensions.xy().as_vec2())
.ceil()
.as_uvec2()
.max(UVec2::ONE);
self.tile_size = tile_size;
self.dimensions = (screen_size.as_vec2() / tile_size.as_vec2())
.ceil()
.as_uvec2()
.extend(requested_dimensions.z)
.max(UVec3::ONE);
debug_assert!(self.dimensions.x * self.dimensions.y * self.dimensions.z <= 4096);
}
fn clear(&mut self) {
self.tile_size = UVec2::ONE;
self.dimensions = UVec3::ZERO;
self.near = 0.0;
self.far = 0.0;
self.clusterable_objects.clear();
}
}
pub fn add_clusters(
mut commands: Commands,
cameras: Query<(Entity, Option<&ClusterConfig>, &Camera), Without<Clusters>>,
) {
for (entity, config, camera) in &cameras {
if !camera.is_active {
continue;
}
let config = config.copied().unwrap_or_default();
commands
.entity(entity)
.insert((Clusters::default(), config));
}
}
impl VisibleClusterableObjects {
#[inline]
pub fn iter(&self) -> impl DoubleEndedIterator<Item = &Entity> {
self.entities.iter()
}
#[inline]
pub fn len(&self) -> usize {
self.entities.len()
}
#[inline]
pub fn is_empty(&self) -> bool {
self.entities.is_empty()
}
}
impl GlobalVisibleClusterableObjects {
#[inline]
pub fn iter(&self) -> impl Iterator<Item = &Entity> {
self.entities.iter()
}
#[inline]
pub fn contains(&self, entity: Entity) -> bool {
self.entities.contains(&entity)
}
}
impl FromWorld for GlobalClusterableObjectMeta {
fn from_world(world: &mut World) -> Self {
Self::new(
world
.resource::<RenderDevice>()
.get_supported_read_only_binding_type(CLUSTERED_FORWARD_STORAGE_BUFFER_COUNT),
)
}
}
impl GlobalClusterableObjectMeta {
pub fn new(buffer_binding_type: BufferBindingType) -> Self {
Self {
gpu_clusterable_objects: GpuClusterableObjects::new(buffer_binding_type),
entity_to_index: EntityHashMap::default(),
}
}
}
impl GpuClusterableObjects {
fn new(buffer_binding_type: BufferBindingType) -> Self {
match buffer_binding_type {
BufferBindingType::Storage { .. } => Self::storage(),
BufferBindingType::Uniform => Self::uniform(),
}
}
fn uniform() -> Self {
Self::Uniform(UniformBuffer::default())
}
fn storage() -> Self {
Self::Storage(StorageBuffer::default())
}
pub(crate) fn set(&mut self, mut clusterable_objects: Vec<GpuClusterableObject>) {
match self {
GpuClusterableObjects::Uniform(buffer) => {
let len = clusterable_objects
.len()
.min(MAX_UNIFORM_BUFFER_CLUSTERABLE_OBJECTS);
let src = &clusterable_objects[..len];
let dst = &mut buffer.get_mut().data[..len];
dst.copy_from_slice(src);
}
GpuClusterableObjects::Storage(buffer) => {
buffer.get_mut().data.clear();
buffer.get_mut().data.append(&mut clusterable_objects);
}
}
}
pub(crate) fn write_buffer(
&mut self,
render_device: &RenderDevice,
render_queue: &RenderQueue,
) {
match self {
GpuClusterableObjects::Uniform(buffer) => {
buffer.write_buffer(render_device, render_queue);
}
GpuClusterableObjects::Storage(buffer) => {
buffer.write_buffer(render_device, render_queue);
}
}
}
pub fn binding(&self) -> Option<BindingResource> {
match self {
GpuClusterableObjects::Uniform(buffer) => buffer.binding(),
GpuClusterableObjects::Storage(buffer) => buffer.binding(),
}
}
pub fn min_size(buffer_binding_type: BufferBindingType) -> NonZeroU64 {
match buffer_binding_type {
BufferBindingType::Storage { .. } => GpuClusterableObjectsStorage::min_size(),
BufferBindingType::Uniform => GpuClusterableObjectsUniform::min_size(),
}
}
}
impl Default for GpuClusterableObjectsUniform {
fn default() -> Self {
Self {
data: Box::new(
[GpuClusterableObject::default(); MAX_UNIFORM_BUFFER_CLUSTERABLE_OBJECTS],
),
}
}
}
#[allow(clippy::too_many_arguments)]
pub(crate) fn clusterable_object_order(
(entity_1, shadows_enabled_1, is_spot_light_1): (&Entity, &bool, &bool),
(entity_2, shadows_enabled_2, is_spot_light_2): (&Entity, &bool, &bool),
) -> std::cmp::Ordering {
is_spot_light_1
.cmp(is_spot_light_2) .then_with(|| shadows_enabled_2.cmp(shadows_enabled_1)) .then_with(|| entity_1.cmp(entity_2)) }
pub fn extract_clusters(
mut commands: Commands,
views: Extract<Query<(Entity, &Clusters, &Camera)>>,
) {
for (entity, clusters, camera) in &views {
if !camera.is_active {
continue;
}
let num_entities: usize = clusters
.clusterable_objects
.iter()
.map(|l| l.entities.len())
.sum();
let mut data = Vec::with_capacity(clusters.clusterable_objects.len() + num_entities);
for cluster_objects in &clusters.clusterable_objects {
data.push(ExtractedClusterableObjectElement::ClusterHeader(
cluster_objects.point_light_count as u32,
cluster_objects.spot_light_count as u32,
));
for clusterable_entity in &cluster_objects.entities {
data.push(ExtractedClusterableObjectElement::ClusterableObjectEntity(
*clusterable_entity,
));
}
}
commands.get_or_spawn(entity).insert((
ExtractedClusterableObjects { data },
ExtractedClusterConfig {
near: clusters.near,
far: clusters.far,
dimensions: clusters.dimensions,
},
));
}
}
pub fn prepare_clusters(
mut commands: Commands,
render_device: Res<RenderDevice>,
render_queue: Res<RenderQueue>,
mesh_pipeline: Res<MeshPipeline>,
global_clusterable_object_meta: Res<GlobalClusterableObjectMeta>,
views: Query<(Entity, &ExtractedClusterableObjects)>,
) {
let render_device = render_device.into_inner();
let supports_storage_buffers = matches!(
mesh_pipeline.clustered_forward_buffer_binding_type,
BufferBindingType::Storage { .. }
);
for (entity, extracted_clusters) in &views {
let mut view_clusters_bindings =
ViewClusterBindings::new(mesh_pipeline.clustered_forward_buffer_binding_type);
view_clusters_bindings.clear();
for record in &extracted_clusters.data {
match record {
ExtractedClusterableObjectElement::ClusterHeader(
point_light_count,
spot_light_count,
) => {
let offset = view_clusters_bindings.n_indices();
view_clusters_bindings.push_offset_and_counts(
offset,
*point_light_count as usize,
*spot_light_count as usize,
);
}
ExtractedClusterableObjectElement::ClusterableObjectEntity(entity) => {
if let Some(clusterable_object_index) =
global_clusterable_object_meta.entity_to_index.get(entity)
{
if view_clusters_bindings.n_indices() >= ViewClusterBindings::MAX_INDICES
&& !supports_storage_buffers
{
warn!(
"Clusterable object index lists are full! The clusterable \
objects in the view are present in too many clusters."
);
break;
}
view_clusters_bindings.push_index(*clusterable_object_index);
}
}
}
}
view_clusters_bindings.write_buffers(render_device, &render_queue);
commands.get_or_spawn(entity).insert(view_clusters_bindings);
}
}
impl ViewClusterBindings {
pub const MAX_OFFSETS: usize = 16384 / 4;
const MAX_UNIFORM_ITEMS: usize = Self::MAX_OFFSETS / 4;
pub const MAX_INDICES: usize = 16384;
pub fn new(buffer_binding_type: BufferBindingType) -> Self {
Self {
n_indices: 0,
n_offsets: 0,
buffers: ViewClusterBuffers::new(buffer_binding_type),
}
}
pub fn clear(&mut self) {
match &mut self.buffers {
ViewClusterBuffers::Uniform {
clusterable_object_index_lists,
cluster_offsets_and_counts,
} => {
*clusterable_object_index_lists.get_mut().data =
[UVec4::ZERO; Self::MAX_UNIFORM_ITEMS];
*cluster_offsets_and_counts.get_mut().data = [UVec4::ZERO; Self::MAX_UNIFORM_ITEMS];
}
ViewClusterBuffers::Storage {
clusterable_object_index_lists,
cluster_offsets_and_counts,
..
} => {
clusterable_object_index_lists.get_mut().data.clear();
cluster_offsets_and_counts.get_mut().data.clear();
}
}
}
pub fn push_offset_and_counts(&mut self, offset: usize, point_count: usize, spot_count: usize) {
match &mut self.buffers {
ViewClusterBuffers::Uniform {
cluster_offsets_and_counts,
..
} => {
let array_index = self.n_offsets >> 2; if array_index >= Self::MAX_UNIFORM_ITEMS {
warn!("cluster offset and count out of bounds!");
return;
}
let component = self.n_offsets & ((1 << 2) - 1);
let packed = pack_offset_and_counts(offset, point_count, spot_count);
cluster_offsets_and_counts.get_mut().data[array_index][component] = packed;
}
ViewClusterBuffers::Storage {
cluster_offsets_and_counts,
..
} => {
cluster_offsets_and_counts.get_mut().data.push(UVec4::new(
offset as u32,
point_count as u32,
spot_count as u32,
0,
));
}
}
self.n_offsets += 1;
}
pub fn n_indices(&self) -> usize {
self.n_indices
}
pub fn push_index(&mut self, index: usize) {
match &mut self.buffers {
ViewClusterBuffers::Uniform {
clusterable_object_index_lists,
..
} => {
let array_index = self.n_indices >> 4; let component = (self.n_indices >> 2) & ((1 << 2) - 1);
let sub_index = self.n_indices & ((1 << 2) - 1);
let index = index as u32;
clusterable_object_index_lists.get_mut().data[array_index][component] |=
index << (8 * sub_index);
}
ViewClusterBuffers::Storage {
clusterable_object_index_lists,
..
} => {
clusterable_object_index_lists
.get_mut()
.data
.push(index as u32);
}
}
self.n_indices += 1;
}
pub fn write_buffers(&mut self, render_device: &RenderDevice, render_queue: &RenderQueue) {
match &mut self.buffers {
ViewClusterBuffers::Uniform {
clusterable_object_index_lists,
cluster_offsets_and_counts,
} => {
clusterable_object_index_lists.write_buffer(render_device, render_queue);
cluster_offsets_and_counts.write_buffer(render_device, render_queue);
}
ViewClusterBuffers::Storage {
clusterable_object_index_lists,
cluster_offsets_and_counts,
} => {
clusterable_object_index_lists.write_buffer(render_device, render_queue);
cluster_offsets_and_counts.write_buffer(render_device, render_queue);
}
}
}
pub fn clusterable_object_index_lists_binding(&self) -> Option<BindingResource> {
match &self.buffers {
ViewClusterBuffers::Uniform {
clusterable_object_index_lists,
..
} => clusterable_object_index_lists.binding(),
ViewClusterBuffers::Storage {
clusterable_object_index_lists,
..
} => clusterable_object_index_lists.binding(),
}
}
pub fn offsets_and_counts_binding(&self) -> Option<BindingResource> {
match &self.buffers {
ViewClusterBuffers::Uniform {
cluster_offsets_and_counts,
..
} => cluster_offsets_and_counts.binding(),
ViewClusterBuffers::Storage {
cluster_offsets_and_counts,
..
} => cluster_offsets_and_counts.binding(),
}
}
pub fn min_size_clusterable_object_index_lists(
buffer_binding_type: BufferBindingType,
) -> NonZeroU64 {
match buffer_binding_type {
BufferBindingType::Storage { .. } => GpuClusterableObjectIndexListsStorage::min_size(),
BufferBindingType::Uniform => GpuClusterableObjectIndexListsUniform::min_size(),
}
}
pub fn min_size_cluster_offsets_and_counts(
buffer_binding_type: BufferBindingType,
) -> NonZeroU64 {
match buffer_binding_type {
BufferBindingType::Storage { .. } => GpuClusterOffsetsAndCountsStorage::min_size(),
BufferBindingType::Uniform => GpuClusterOffsetsAndCountsUniform::min_size(),
}
}
}
impl ViewClusterBuffers {
fn new(buffer_binding_type: BufferBindingType) -> Self {
match buffer_binding_type {
BufferBindingType::Storage { .. } => Self::storage(),
BufferBindingType::Uniform => Self::uniform(),
}
}
fn uniform() -> Self {
ViewClusterBuffers::Uniform {
clusterable_object_index_lists: UniformBuffer::default(),
cluster_offsets_and_counts: UniformBuffer::default(),
}
}
fn storage() -> Self {
ViewClusterBuffers::Storage {
clusterable_object_index_lists: StorageBuffer::default(),
cluster_offsets_and_counts: StorageBuffer::default(),
}
}
}
fn pack_offset_and_counts(offset: usize, point_count: usize, spot_count: usize) -> u32 {
((offset as u32 & CLUSTER_OFFSET_MASK) << (CLUSTER_COUNT_SIZE * 2))
| (point_count as u32 & CLUSTER_COUNT_MASK) << CLUSTER_COUNT_SIZE
| (spot_count as u32 & CLUSTER_COUNT_MASK)
}
#[derive(ShaderType)]
struct GpuClusterableObjectIndexListsUniform {
data: Box<[UVec4; ViewClusterBindings::MAX_UNIFORM_ITEMS]>,
}
const _: () = assert!(GpuClusterableObjectIndexListsUniform::SHADER_SIZE.get() <= 16384);
impl Default for GpuClusterableObjectIndexListsUniform {
fn default() -> Self {
Self {
data: Box::new([UVec4::ZERO; ViewClusterBindings::MAX_UNIFORM_ITEMS]),
}
}
}
impl Default for GpuClusterOffsetsAndCountsUniform {
fn default() -> Self {
Self {
data: Box::new([UVec4::ZERO; ViewClusterBindings::MAX_UNIFORM_ITEMS]),
}
}
}