Implement custom shader for window-open

3 files changed, 189 insertions, 42 deletions

diff --git a/src/layout/mod.rs b/src/layout/mod.rs
index ec02bfc6..f4b88560 100644
--- a/src/layout/mod.rs
+++ b/src/layout/mod.rs
@@ -58,6 +58,7 @@ use crate::window::ResolvedWindowRules;
 pub mod closing_window;
 pub mod focus_ring;
 pub mod monitor;
+pub mod opening_window;
 pub mod tile;
 pub mod workspace;
 
diff --git a/src/layout/opening_window.rs b/src/layout/opening_window.rs
new file mode 100644
index 00000000..1a8b3b99
--- /dev/null
+++ b/src/layout/opening_window.rs
@@ -0,0 +1,156 @@
+use std::collections::HashMap;
+use std::time::Duration;
+
+use anyhow::Context as _;
+use glam::{Mat3, Vec2};
+use smithay::backend::allocator::Fourcc;
+use smithay::backend::renderer::element::texture::TextureRenderElement;
+use smithay::backend::renderer::element::utils::{
+    Relocate, RelocateRenderElement, RescaleRenderElement,
+};
+use smithay::backend::renderer::element::{Id, Kind, RenderElement};
+use smithay::backend::renderer::gles::{GlesRenderer, Uniform};
+use smithay::backend::renderer::{Renderer as _, Texture};
+use smithay::utils::{Logical, Point, Rectangle, Scale, Size, Transform};
+
+use crate::animation::Animation;
+use crate::niri_render_elements;
+use crate::render_helpers::primary_gpu_texture::PrimaryGpuTextureRenderElement;
+use crate::render_helpers::render_to_encompassing_texture;
+use crate::render_helpers::shader_element::ShaderRenderElement;
+use crate::render_helpers::shaders::{mat3_uniform, ProgramType, Shaders};
+
+#[derive(Debug)]
+pub struct OpenAnimation {
+    anim: Animation,
+    random_seed: f32,
+}
+
+niri_render_elements! {
+    OpeningWindowRenderElement => {
+        Texture = RelocateRenderElement<RescaleRenderElement<PrimaryGpuTextureRenderElement>>,
+        Shader = ShaderRenderElement,
+    }
+}
+
+impl OpenAnimation {
+    pub fn new(anim: Animation) -> Self {
+        Self {
+            anim,
+            random_seed: fastrand::f32(),
+        }
+    }
+
+    pub fn advance_animations(&mut self, current_time: Duration) {
+        self.anim.set_current_time(current_time);
+    }
+
+    pub fn is_done(&self) -> bool {
+        self.anim.is_done()
+    }
+
+    // We can't depend on view_rect here, because the result of window opening can be snapshot and
+    // then rendered elsewhere.
+    pub fn render(
+        &self,
+        renderer: &mut GlesRenderer,
+        elements: &[impl RenderElement<GlesRenderer>],
+        geo_size: Size<i32, Logical>,
+        location: Point<i32, Logical>,
+        scale: Scale<f64>,
+    ) -> anyhow::Result<OpeningWindowRenderElement> {
+        let progress = self.anim.value();
+        let clamped_progress = self.anim.clamped_value().clamp(0., 1.);
+
+        let (texture, _sync_point, geo) = render_to_encompassing_texture(
+            renderer,
+            scale,
+            Transform::Normal,
+            Fourcc::Abgr8888,
+            elements,
+        )
+        .context("error rendering to texture")?;
+
+        let offset = geo.loc.to_f64().to_logical(scale);
+        let texture_size = geo.size.to_f64().to_logical(scale);
+
+        if Shaders::get(renderer).program(ProgramType::Open).is_some() {
+            let mut area = Rectangle::from_loc_and_size(location.to_f64() + offset, texture_size);
+
+            // Expand the area a bit to allow for more varied effects.
+            let mut target_size = area.size.upscale(1.5);
+            target_size.w = f64::max(area.size.w + 1000., target_size.w);
+            target_size.h = f64::max(area.size.h + 1000., target_size.h);
+            let diff = target_size.to_point() - area.size.to_point();
+            area.loc -= diff.downscale(2.);
+            area.size += diff.to_size();
+
+            let area = area.to_i32_up();
+            let area_loc = Vec2::new(area.loc.x as f32, area.loc.y as f32);
+            let area_size = Vec2::new(area.size.w as f32, area.size.h as f32);
+
+            let geo_loc = Vec2::new(location.x as f32, location.y as f32);
+            let geo_size = Vec2::new(geo_size.w as f32, geo_size.h as f32);
+
+            let input_to_geo = Mat3::from_scale(area_size / geo_size)
+                * Mat3::from_translation((area_loc - geo_loc) / area_size);
+
+            let tex_scale = Vec2::new(scale.x as f32, scale.y as f32);
+            let tex_loc = Vec2::new(offset.x as f32, offset.y as f32);
+            let tex_size = Vec2::new(texture.width() as f32, texture.height() as f32) / tex_scale;
+
+            let geo_to_tex =
+                Mat3::from_translation(-tex_loc / tex_size) * Mat3::from_scale(geo_size / tex_size);
+
+            return Ok(ShaderRenderElement::new(
+                ProgramType::Open,
+                area.size,
+                None,
+                1.,
+                vec![
+                    mat3_uniform("niri_input_to_geo", input_to_geo),
+                    Uniform::new("niri_geo_size", geo_size.to_array()),
+                    mat3_uniform("niri_geo_to_tex", geo_to_tex),
+                    Uniform::new("niri_progress", progress as f32),
+                    Uniform::new("niri_clamped_progress", clamped_progress as f32),
+                    Uniform::new("niri_random_seed", self.random_seed),
+                ],
+                HashMap::from([(String::from("niri_tex"), texture.clone())]),
+                Kind::Unspecified,
+            )
+            .with_location(area.loc)
+            .into());
+        }
+
+        let elem = TextureRenderElement::from_static_texture(
+            Id::new(),
+            renderer.id(),
+            Point::from((0., 0.)),
+            texture.clone(),
+            scale.x as i32,
+            Transform::Normal,
+            Some(clamped_progress as f32),
+            None,
+            None,
+            None,
+            Kind::Unspecified,
+        );
+
+        let elem = PrimaryGpuTextureRenderElement(elem);
+
+        let center = geo_size.to_point().to_f64().downscale(2.);
+        let elem = RescaleRenderElement::from_element(
+            elem,
+            (center - offset).to_physical_precise_round(scale),
+            (progress / 2. + 0.5).max(0.),
+        );
+
+        let elem = RelocateRenderElement::from_element(
+            elem,
+            (location.to_f64() + offset).to_physical_precise_round(scale),
+            Relocate::Relative,
+        );
+
+        Ok(elem.into())
+    }
+}
diff --git a/src/layout/tile.rs b/src/layout/tile.rs
index d53eb2b1..719f12bc 100644
--- a/src/layout/tile.rs
+++ b/src/layout/tile.rs
@@ -5,12 +5,12 @@ use std::time::Duration;
 use niri_config::CornerRadius;
 use smithay::backend::allocator::Fourcc;
 use smithay::backend::renderer::element::solid::{SolidColorBuffer, SolidColorRenderElement};
-use smithay::backend::renderer::element::utils::RescaleRenderElement;
 use smithay::backend::renderer::element::{Element, Kind};
 use smithay::backend::renderer::gles::GlesRenderer;
 use smithay::utils::{Logical, Point, Rectangle, Scale, Size, Transform};
 
 use super::focus_ring::{FocusRing, FocusRingRenderElement};
+use super::opening_window::{OpenAnimation, OpeningWindowRenderElement};
 use super::{
     LayoutElement, LayoutElementRenderElement, LayoutElementRenderSnapshot, Options,
     RESIZE_ANIMATION_THRESHOLD,
@@ -20,7 +20,6 @@ use crate::niri_render_elements;
 use crate::render_helpers::border::BorderRenderElement;
 use crate::render_helpers::clipped_surface::{ClippedSurfaceRenderElement, RoundedCornerDamage};
 use crate::render_helpers::damage::ExtraDamage;
-use crate::render_helpers::offscreen::OffscreenRenderElement;
 use crate::render_helpers::renderer::NiriRenderer;
 use crate::render_helpers::resize::ResizeRenderElement;
 use crate::render_helpers::snapshot::RenderSnapshot;
@@ -54,7 +53,7 @@ pub struct Tile<W: LayoutElement> {
     fullscreen_size: Size<i32, Logical>,
 
     /// The animation upon opening a window.
-    open_animation: Option<Animation>,
+    open_animation: Option<OpenAnimation>,
 
     /// The animation of the window resizing.
     resize_animation: Option<ResizeAnimation>,
@@ -80,7 +79,7 @@ niri_render_elements! {
         LayoutElement = LayoutElementRenderElement<R>,
         FocusRing = FocusRingRenderElement,
         SolidColor = SolidColorRenderElement,
-        Offscreen = RescaleRenderElement<OffscreenRenderElement>,
+        Opening = OpeningWindowRenderElement,
         Resize = ResizeRenderElement,
         Border = BorderRenderElement,
         ClippedSurface = ClippedSurfaceRenderElement<R>,
@@ -201,9 +200,9 @@ impl<W: LayoutElement> Tile<W> {
     }
 
     pub fn advance_animations(&mut self, current_time: Duration) {
-        if let Some(anim) = &mut self.open_animation {
-            anim.set_current_time(current_time);
-            if anim.is_done() {
+        if let Some(open) = &mut self.open_animation {
+            open.advance_animations(current_time);
+            if open.is_done() {
                 self.open_animation = None;
             }
         }
@@ -299,16 +298,12 @@ impl<W: LayoutElement> Tile<W> {
     }
 
     pub fn start_open_animation(&mut self) {
-        self.open_animation = Some(Animation::new(
+        self.open_animation = Some(OpenAnimation::new(Animation::new(
             0.,
             1.,
             0.,
-            self.options.animations.window_open.0,
-        ));
-    }
-
-    pub fn open_animation(&self) -> &Option<Animation> {
-        &self.open_animation
+            self.options.animations.window_open.anim,
+        )));
     }
 
     pub fn resize_animation(&self) -> Option<&Animation> {
@@ -797,39 +792,34 @@ impl<W: LayoutElement> Tile<W> {
     ) -> impl Iterator<Item = TileRenderElement<R>> {
         let _span = tracy_client::span!("Tile::render");
 
-        if let Some(anim) = &self.open_animation {
+        let mut open_anim_elem = None;
+        let mut window_elems = None;
+
+        if let Some(open) = &self.open_animation {
             let renderer = renderer.as_gles_renderer();
-            let elements = self.render_inner(renderer, location, scale, focus_ring, target);
+            let elements =
+                self.render_inner(renderer, Point::from((0, 0)), scale, focus_ring, target);
             let elements = elements.collect::<Vec<TileRenderElement<_>>>();
+            match open.render(renderer, &elements, self.tile_size(), location, scale) {
+                Ok(elem) => {
+                    self.window()
+                        .set_offscreen_element_id(Some(elem.id().clone()));
+                    open_anim_elem = Some(elem.into());
+                }
+                Err(err) => {
+                    warn!("error rendering window opening animation: {err:?}");
+                }
+            }
+        }
 
-            let elem = OffscreenRenderElement::new(
-                renderer,
-                scale.x as i32,
-                &elements,
-                anim.clamped_value().clamp(0., 1.) as f32,
-            );
-            self.window()
-                .set_offscreen_element_id(Some(elem.id().clone()));
-
-            let mut center = location;
-            center.x += self.tile_size().w / 2;
-            center.y += self.tile_size().h / 2;
-
-            Some(TileRenderElement::Offscreen(
-                RescaleRenderElement::from_element(
-                    elem,
-                    center.to_physical_precise_round(scale),
-                    (anim.value() / 2. + 0.5).max(0.),
-                ),
-            ))
-            .into_iter()
-            .chain(None.into_iter().flatten())
-        } else {
+        if open_anim_elem.is_none() {
             self.window().set_offscreen_element_id(None);
-
-            let elements = self.render_inner(renderer, location, scale, focus_ring, target);
-            None.into_iter().chain(Some(elements).into_iter().flatten())
+            window_elems = Some(self.render_inner(renderer, location, scale, focus_ring, target));
         }
+
+        open_anim_elem
+            .into_iter()
+            .chain(window_elems.into_iter().flatten())
     }
 
     pub fn store_unmap_snapshot_if_empty(