Implement rounded window corners

1 files changed, 269 insertions, 0 deletions

diff --git a/src/render_helpers/clipped_surface.rs b/src/render_helpers/clipped_surface.rs
new file mode 100644
index 00000000..69c2a012
--- /dev/null
+++ b/src/render_helpers/clipped_surface.rs
@@ -0,0 +1,269 @@
+use glam::{Mat3, Vec2};
+use niri_config::CornerRadius;
+use smithay::backend::renderer::element::surface::WaylandSurfaceRenderElement;
+use smithay::backend::renderer::element::{Element, Id, Kind, RenderElement, UnderlyingStorage};
+use smithay::backend::renderer::gles::{
+    GlesError, GlesFrame, GlesRenderer, GlesTexProgram, Uniform,
+};
+use smithay::backend::renderer::utils::{CommitCounter, DamageSet};
+use smithay::utils::{Buffer, Logical, Physical, Rectangle, Scale, Transform};
+
+use super::renderer::{AsGlesFrame as _, NiriRenderer};
+use super::shaders::{mat3_uniform, Shaders};
+use crate::backend::tty::{TtyFrame, TtyRenderer, TtyRendererError};
+
+#[derive(Debug)]
+pub struct ClippedSurfaceRenderElement<R: NiriRenderer> {
+    inner: WaylandSurfaceRenderElement<R>,
+    program: GlesTexProgram,
+    corner_radius: CornerRadius,
+    geometry: Rectangle<i32, Logical>,
+    input_to_geo: Mat3,
+}
+
+impl<R: NiriRenderer> ClippedSurfaceRenderElement<R> {
+    pub fn new(
+        elem: WaylandSurfaceRenderElement<R>,
+        scale: Scale<f64>,
+        geometry: Rectangle<i32, Logical>,
+        program: GlesTexProgram,
+        corner_radius: CornerRadius,
+    ) -> Self {
+        let elem_geo = elem.geometry(scale);
+
+        let elem_geo_loc = Vec2::new(elem_geo.loc.x as f32, elem_geo.loc.y as f32);
+        let elem_geo_size = Vec2::new(elem_geo.size.w as f32, elem_geo.size.h as f32);
+
+        let geo = geometry.to_physical_precise_round(scale);
+        let geo_loc = Vec2::new(geo.loc.x, geo.loc.y);
+        let geo_size = Vec2::new(geo.size.w, geo.size.h);
+
+        let buf_size = elem.buffer_size().unwrap();
+        let buf_size = Vec2::new(buf_size.w as f32, buf_size.h as f32);
+
+        let view = elem.view().unwrap();
+        let src_loc = Vec2::new(view.src.loc.x as f32, view.src.loc.y as f32);
+        let src_size = Vec2::new(view.src.size.w as f32, view.src.size.h as f32);
+
+        let transform = elem.transform();
+        // HACK: ??? for some reason flipped ones are fine.
+        let transform = match transform {
+            Transform::_90 => Transform::_270,
+            Transform::_270 => Transform::_90,
+            x => x,
+        };
+        let transform_matrix = Mat3::from_translation(Vec2::new(0.5, 0.5))
+            * Mat3::from_cols_array(transform.matrix().as_ref())
+            * Mat3::from_translation(-Vec2::new(0.5, 0.5));
+
+        // FIXME: y_inverted
+        let input_to_geo = transform_matrix * Mat3::from_scale(elem_geo_size / geo_size)
+            * Mat3::from_translation((elem_geo_loc - geo_loc) / elem_geo_size)
+            // Apply viewporter src.
+            * Mat3::from_scale(buf_size / src_size)
+            * Mat3::from_translation(-src_loc / buf_size);
+
+        Self {
+            inner: elem,
+            program,
+            corner_radius,
+            geometry,
+            input_to_geo,
+        }
+    }
+
+    pub fn shader(renderer: &mut R) -> Option<&GlesTexProgram> {
+        Shaders::get(renderer).clipped_surface.as_ref()
+    }
+
+    pub fn will_clip(
+        elem: &WaylandSurfaceRenderElement<R>,
+        scale: Scale<f64>,
+        geometry: Rectangle<i32, Logical>,
+        corner_radius: CornerRadius,
+    ) -> bool {
+        let elem_geo = elem.geometry(scale);
+        let geo = geometry.to_physical_precise_round(scale);
+
+        if corner_radius == CornerRadius::default() {
+            !geo.contains_rect(elem_geo)
+        } else {
+            let corners = Self::rounded_corners(geometry.to_f64(), corner_radius);
+            let corners = corners
+                .into_iter()
+                .map(|rect| rect.to_physical_precise_round(scale));
+            let geo = Rectangle::subtract_rects_many([geo], corners);
+            !Rectangle::subtract_rects_many([elem_geo], geo).is_empty()
+        }
+    }
+
+    fn rounded_corners(
+        geo: Rectangle<f64, Logical>,
+        corner_radius: CornerRadius,
+    ) -> [Rectangle<f64, Logical>; 4] {
+        let top_left = corner_radius.top_left as f64;
+        let top_right = corner_radius.top_right as f64;
+        let bottom_right = corner_radius.bottom_right as f64;
+        let bottom_left = corner_radius.bottom_left as f64;
+
+        [
+            Rectangle::from_loc_and_size(geo.loc, (top_left, top_left)),
+            Rectangle::from_loc_and_size(
+                (geo.loc.x + geo.size.w - top_right, geo.loc.y),
+                (top_right, top_right),
+            ),
+            Rectangle::from_loc_and_size(
+                (
+                    geo.loc.x + geo.size.w - bottom_right,
+                    geo.loc.y + geo.size.h - bottom_right,
+                ),
+                (bottom_right, bottom_right),
+            ),
+            Rectangle::from_loc_and_size(
+                (geo.loc.x, geo.loc.y + geo.size.h - bottom_left),
+                (bottom_left, bottom_left),
+            ),
+        ]
+    }
+}
+
+impl<R: NiriRenderer> Element for ClippedSurfaceRenderElement<R> {
+    fn id(&self) -> &Id {
+        self.inner.id()
+    }
+
+    fn current_commit(&self) -> CommitCounter {
+        self.inner.current_commit()
+    }
+
+    fn geometry(&self, scale: Scale<f64>) -> Rectangle<i32, Physical> {
+        self.inner.geometry(scale)
+    }
+
+    fn src(&self) -> Rectangle<f64, Buffer> {
+        self.inner.src()
+    }
+
+    fn transform(&self) -> Transform {
+        self.inner.transform()
+    }
+
+    fn damage_since(
+        &self,
+        scale: Scale<f64>,
+        commit: Option<CommitCounter>,
+    ) -> DamageSet<i32, Physical> {
+        // FIXME: radius changes need to cause damage.
+        let damage = self.inner.damage_since(scale, commit);
+
+        // Intersect with geometry, since we're clipping by it.
+        let mut geo = self.geometry.to_physical_precise_round(scale);
+        geo.loc -= self.geometry(scale).loc;
+        damage
+            .into_iter()
+            .filter_map(|rect| rect.intersection(geo))
+            .collect()
+    }
+
+    fn opaque_regions(&self, scale: Scale<f64>) -> Vec<Rectangle<i32, Physical>> {
+        let regions = self.inner.opaque_regions(scale);
+
+        // Intersect with geometry, since we're clipping by it.
+        let mut geo = self.geometry.to_physical_precise_round(scale);
+        geo.loc -= self.geometry(scale).loc;
+        let regions = regions
+            .into_iter()
+            .filter_map(|rect| rect.intersection(geo));
+
+        // Subtract the rounded corners.
+        if self.corner_radius == CornerRadius::default() {
+            regions.collect()
+        } else {
+            let corners = Self::rounded_corners(self.geometry.to_f64(), self.corner_radius);
+
+            let elem_loc = self.geometry(scale).loc;
+            let corners = corners.into_iter().map(|rect| {
+                let mut rect = rect.to_physical_precise_round(scale);
+                rect.loc -= elem_loc;
+                rect
+            });
+
+            Rectangle::subtract_rects_many(regions, corners)
+        }
+    }
+
+    fn alpha(&self) -> f32 {
+        self.inner.alpha()
+    }
+
+    fn kind(&self) -> Kind {
+        self.inner.kind()
+    }
+}
+
+impl RenderElement<GlesRenderer> for ClippedSurfaceRenderElement<GlesRenderer> {
+    fn draw(
+        &self,
+        frame: &mut GlesFrame<'_>,
+        src: Rectangle<f64, Buffer>,
+        dst: Rectangle<i32, Physical>,
+        damage: &[Rectangle<i32, Physical>],
+    ) -> Result<(), GlesError> {
+        frame.override_default_tex_program(
+            self.program.clone(),
+            vec![
+                Uniform::new(
+                    "geo_size",
+                    (self.geometry.size.w as f32, self.geometry.size.h as f32),
+                ),
+                Uniform::new("corner_radius", <[f32; 4]>::from(self.corner_radius)),
+                mat3_uniform("input_to_geo", self.input_to_geo),
+            ],
+        );
+        RenderElement::<GlesRenderer>::draw(&self.inner, frame, src, dst, damage)?;
+        frame.clear_tex_program_override();
+        Ok(())
+    }
+
+    fn underlying_storage(&self, _renderer: &mut GlesRenderer) -> Option<UnderlyingStorage> {
+        // If scanout for things other than Wayland buffers is implemented, this will need to take
+        // the target GPU into account.
+        None
+    }
+}
+
+impl<'render> RenderElement<TtyRenderer<'render>>
+    for ClippedSurfaceRenderElement<TtyRenderer<'render>>
+{
+    fn draw(
+        &self,
+        frame: &mut TtyFrame<'render, '_>,
+        src: Rectangle<f64, Buffer>,
+        dst: Rectangle<i32, Physical>,
+        damage: &[Rectangle<i32, Physical>],
+    ) -> Result<(), TtyRendererError<'render>> {
+        frame.as_gles_frame().override_default_tex_program(
+            self.program.clone(),
+            vec![
+                Uniform::new(
+                    "geo_size",
+                    (self.geometry.size.w as f32, self.geometry.size.h as f32),
+                ),
+                Uniform::new("corner_radius", <[f32; 4]>::from(self.corner_radius)),
+                mat3_uniform("input_to_geo", self.input_to_geo),
+            ],
+        );
+        RenderElement::draw(&self.inner, frame, src, dst, damage)?;
+        frame.as_gles_frame().clear_tex_program_override();
+        Ok(())
+    }
+
+    fn underlying_storage(
+        &self,
+        _renderer: &mut TtyRenderer<'render>,
+    ) -> Option<UnderlyingStorage> {
+        // If scanout for things other than Wayland buffers is implemented, this will need to take
+        // the target GPU into account.
+        None
+    }
+}