Implement rounded window corners

8 files changed, 287 insertions, 88 deletions

diff --git a/src/render_helpers/shaders/border.frag b/src/render_helpers/shaders/border.frag
new file mode 100644
index 00000000..89edb363
--- /dev/null
+++ b/src/render_helpers/shaders/border.frag
@@ -0,0 +1,87 @@
+precision mediump float;
+
+#if defined(DEBUG_FLAGS)
+uniform float niri_tint;
+#endif
+
+uniform float niri_alpha;
+
+uniform vec2 niri_size;
+varying vec2 niri_v_coords;
+
+uniform vec4 color_from;
+uniform vec4 color_to;
+uniform vec2 grad_offset;
+uniform float grad_width;
+uniform vec2 grad_vec;
+
+uniform mat3 input_to_geo;
+uniform vec2 geo_size;
+uniform vec4 outer_radius;
+uniform float border_width;
+
+vec4 gradient_color(vec2 coords) {
+    coords = coords + grad_offset;
+
+    if ((grad_vec.x < 0.0 && 0.0 <= grad_vec.y) || (0.0 <= grad_vec.x && grad_vec.y < 0.0))
+        coords.x -= grad_width;
+
+    float frac = dot(coords, grad_vec) / dot(grad_vec, grad_vec);
+
+    if (grad_vec.y < 0.0)
+        frac += 1.0;
+
+    frac = clamp(frac, 0.0, 1.0);
+    return mix(color_from, color_to, frac);
+}
+
+float rounding_alpha(vec2 coords, vec2 size, vec4 corner_radius) {
+    vec2 center;
+    float radius;
+
+    if (coords.x < corner_radius.x && coords.y < corner_radius.x) {
+        radius = corner_radius.x;
+        center = vec2(radius, radius);
+    } else if (size.x - corner_radius.y < coords.x && coords.y < corner_radius.y) {
+        radius = corner_radius.y;
+        center = vec2(size.x - radius, radius);
+    } else if (size.x - corner_radius.z < coords.x && size.y - corner_radius.z < coords.y) {
+        radius = corner_radius.z;
+        center = vec2(size.x - radius, size.y - radius);
+    } else if (coords.x < corner_radius.w && size.y - corner_radius.w < coords.y) {
+        radius = corner_radius.w;
+        center = vec2(radius, size.y - radius);
+    } else {
+        return 1.0;
+    }
+
+    float dist = distance(coords, center);
+    return 1.0 - smoothstep(radius - 0.5, radius + 0.5, dist);
+}
+
+void main() {
+    vec4 color = gradient_color(niri_v_coords * niri_size);
+
+    vec3 coords_geo = input_to_geo * vec3(niri_v_coords, 1.0);
+    color = color * rounding_alpha(coords_geo.xy, geo_size, outer_radius);
+
+    if (border_width > 0.0) {
+        coords_geo -= vec3(border_width);
+        vec2 inner_geo_size = geo_size - vec2(border_width * 2.0);
+        if (0.0 <= coords_geo.x && coords_geo.x <= inner_geo_size.x
+                && 0.0 <= coords_geo.y && coords_geo.y <= inner_geo_size.y)
+        {
+            vec4 inner_radius = max(outer_radius - vec4(border_width), 0.0);
+            color = color * (1.0 - rounding_alpha(coords_geo.xy, inner_geo_size, inner_radius));
+        }
+    }
+
+    color = color * niri_alpha;
+
+#if defined(DEBUG_FLAGS)
+    if (niri_tint == 1.0)
+        color = vec4(0.0, 0.2, 0.0, 0.2) + color * 0.8;
+#endif
+
+    gl_FragColor = color;
+}
diff --git a/src/render_helpers/shaders/clipped_surface.frag b/src/render_helpers/shaders/clipped_surface.frag
new file mode 100644
index 00000000..7bee28b1
--- /dev/null
+++ b/src/render_helpers/shaders/clipped_surface.frag
@@ -0,0 +1,77 @@
+#version 100
+
+//_DEFINES_
+
+#if defined(EXTERNAL)
+#extension GL_OES_EGL_image_external : require
+#endif
+
+precision mediump float;
+#if defined(EXTERNAL)
+uniform samplerExternalOES tex;
+#else
+uniform sampler2D tex;
+#endif
+
+uniform float alpha;
+varying vec2 v_coords;
+
+#if defined(DEBUG_FLAGS)
+uniform float tint;
+#endif
+
+uniform vec2 geo_size;
+uniform vec4 corner_radius;
+uniform mat3 input_to_geo;
+
+float rounding_alpha(vec2 coords, vec2 size) {
+    vec2 center;
+    float radius;
+
+    if (coords.x < corner_radius.x && coords.y < corner_radius.x) {
+        radius = corner_radius.x;
+        center = vec2(radius, radius);
+    } else if (size.x - corner_radius.y < coords.x && coords.y < corner_radius.y) {
+        radius = corner_radius.y;
+        center = vec2(size.x - radius, radius);
+    } else if (size.x - corner_radius.z < coords.x && size.y - corner_radius.z < coords.y) {
+        radius = corner_radius.z;
+        center = vec2(size.x - radius, size.y - radius);
+    } else if (coords.x < corner_radius.w && size.y - corner_radius.w < coords.y) {
+        radius = corner_radius.w;
+        center = vec2(radius, size.y - radius);
+    } else {
+        return 1.0;
+    }
+
+    float dist = distance(coords, center);
+    return 1.0 - smoothstep(radius - 0.5, radius + 0.5, dist);
+}
+
+void main() {
+    vec3 coords_geo = input_to_geo * vec3(v_coords, 1.0);
+
+    // Sample the texture.
+    vec4 color = texture2D(tex, v_coords);
+#if defined(NO_ALPHA)
+    color = vec4(color.rgb, 1.0);
+#endif
+
+    if (coords_geo.x < 0.0 || 1.0 < coords_geo.x || coords_geo.y < 0.0 || 1.0 < coords_geo.y) {
+        // Clip outside geometry.
+        color = vec4(0.0);
+    } else {
+        // Apply corner rounding inside geometry.
+        color = color * rounding_alpha(coords_geo.xy * geo_size, geo_size);
+    }
+
+    // Apply final alpha and tint.
+    color = color * alpha;
+
+#if defined(DEBUG_FLAGS)
+    if (tint == 1.0)
+        color = vec4(0.0, 0.2, 0.0, 0.2) + color * 0.8;
+#endif
+
+    gl_FragColor = color;
+}
diff --git a/src/render_helpers/shaders/gradient_border.frag b/src/render_helpers/shaders/gradient_border.frag
deleted file mode 100644
index 67114eb4..00000000
--- a/src/render_helpers/shaders/gradient_border.frag
+++ /dev/null
@@ -1,35 +0,0 @@
-precision mediump float;
-uniform float alpha;
-#if defined(DEBUG_FLAGS)
-uniform float tint;
-#endif
-uniform vec2 size;
-varying vec2 v_coords;
-
-uniform vec4 color_from;
-uniform vec4 color_to;
-uniform vec2 grad_offset;
-uniform float grad_width;
-uniform vec2 grad_vec;
-
-void main() {
-    vec2 coords = v_coords * size + grad_offset;
-
-    if ((grad_vec.x < 0.0 && 0.0 <= grad_vec.y) || (0.0 <= grad_vec.x && grad_vec.y < 0.0))
-        coords.x -= grad_width;
-
-    float frac = dot(coords, grad_vec) / dot(grad_vec, grad_vec);
-
-    if (grad_vec.y < 0.0)
-        frac += 1.0;
-
-    frac = clamp(frac, 0.0, 1.0);
-    vec4 out_color = mix(color_from, color_to, frac);
-
-#if defined(DEBUG_FLAGS)
-    if (tint == 1.0)
-        out_color = vec4(0.0, 0.3, 0.0, 0.2) + out_color * 0.8;
-#endif
-
-    gl_FragColor = out_color;
-}
diff --git a/src/render_helpers/shaders/mod.rs b/src/render_helpers/shaders/mod.rs
index 4c4530c5..5937fcc1 100644
--- a/src/render_helpers/shaders/mod.rs
+++ b/src/render_helpers/shaders/mod.rs
@@ -2,35 +2,55 @@ use std::cell::RefCell;
 
 use glam::Mat3;
 use smithay::backend::renderer::gles::{
-    GlesError, GlesPixelProgram, GlesRenderer, Uniform, UniformName, UniformType, UniformValue,
+    GlesError, GlesRenderer, GlesTexProgram, Uniform, UniformName, UniformType, UniformValue,
 };
 
-use super::primary_gpu_pixel_shader_with_textures::PixelWithTexturesProgram;
 use super::renderer::NiriRenderer;
+use super::shader_element::ShaderProgram;
 
 pub struct Shaders {
-    pub gradient_border: Option<GlesPixelProgram>,
-    pub resize: Option<PixelWithTexturesProgram>,
-    pub custom_resize: RefCell<Option<PixelWithTexturesProgram>>,
+    pub border: Option<ShaderProgram>,
+    pub clipped_surface: Option<GlesTexProgram>,
+    pub resize: Option<ShaderProgram>,
+    pub custom_resize: RefCell<Option<ShaderProgram>>,
 }
 
 impl Shaders {
     fn compile(renderer: &mut GlesRenderer) -> Self {
         let _span = tracy_client::span!("Shaders::compile");
 
-        let gradient_border = renderer
-            .compile_custom_pixel_shader(
-                include_str!("gradient_border.frag"),
+        let border = ShaderProgram::compile(
+            renderer,
+            include_str!("border.frag"),
+            &[
+                UniformName::new("color_from", UniformType::_4f),
+                UniformName::new("color_to", UniformType::_4f),
+                UniformName::new("grad_offset", UniformType::_2f),
+                UniformName::new("grad_width", UniformType::_1f),
+                UniformName::new("grad_vec", UniformType::_2f),
+                UniformName::new("input_to_geo", UniformType::Matrix3x3),
+                UniformName::new("geo_size", UniformType::_2f),
+                UniformName::new("outer_radius", UniformType::_4f),
+                UniformName::new("border_width", UniformType::_1f),
+            ],
+            &[],
+        )
+        .map_err(|err| {
+            warn!("error compiling border shader: {err:?}");
+        })
+        .ok();
+
+        let clipped_surface = renderer
+            .compile_custom_texture_shader(
+                include_str!("clipped_surface.frag"),
                 &[
-                    UniformName::new("color_from", UniformType::_4f),
-                    UniformName::new("color_to", UniformType::_4f),
-                    UniformName::new("grad_offset", UniformType::_2f),
-                    UniformName::new("grad_width", UniformType::_1f),
-                    UniformName::new("grad_vec", UniformType::_2f),
+                    UniformName::new("geo_size", UniformType::_2f),
+                    UniformName::new("corner_radius", UniformType::_4f),
+                    UniformName::new("input_to_geo", UniformType::Matrix3x3),
                 ],
             )
             .map_err(|err| {
-                warn!("error compiling gradient border shader: {err:?}");
+                warn!("error compiling clipped surface shader: {err:?}");
             })
             .ok();
 
@@ -41,7 +61,8 @@ impl Shaders {
             .ok();
 
         Self {
-            gradient_border,
+            border,
+            clipped_surface,
             resize,
             custom_resize: RefCell::new(None),
         }
@@ -56,12 +77,12 @@ impl Shaders {
 
     pub fn replace_custom_resize_program(
         &self,
-        program: Option<PixelWithTexturesProgram>,
-    ) -> Option<PixelWithTexturesProgram> {
+        program: Option<ShaderProgram>,
+    ) -> Option<ShaderProgram> {
         self.custom_resize.replace(program)
     }
 
-    pub fn resize(&self) -> Option<PixelWithTexturesProgram> {
+    pub fn resize(&self) -> Option<ShaderProgram> {
         self.custom_resize
             .borrow()
             .clone()
@@ -80,12 +101,12 @@ pub fn init(renderer: &mut GlesRenderer) {
 fn compile_resize_program(
     renderer: &mut GlesRenderer,
     src: &str,
-) -> Result<PixelWithTexturesProgram, GlesError> {
-    let mut program = include_str!("resize-prelude.frag").to_string();
+) -> Result<ShaderProgram, GlesError> {
+    let mut program = include_str!("resize_prelude.frag").to_string();
     program.push_str(src);
-    program.push_str(include_str!("resize-epilogue.frag"));
+    program.push_str(include_str!("resize_epilogue.frag"));
 
-    PixelWithTexturesProgram::compile(
+    ShaderProgram::compile(
         renderer,
         &program,
         &[
@@ -97,6 +118,8 @@ fn compile_resize_program(
             UniformName::new("niri_geo_to_tex_next", UniformType::Matrix3x3),
             UniformName::new("niri_progress", UniformType::_1f),
             UniformName::new("niri_clamped_progress", UniformType::_1f),
+            UniformName::new("niri_corner_radius", UniformType::_4f),
+            UniformName::new("niri_clip_to_geometry", UniformType::_1f),
         ],
         &["niri_tex_prev", "niri_tex_next"],
     )
diff --git a/src/render_helpers/shaders/resize-epilogue.frag b/src/render_helpers/shaders/resize-epilogue.frag
deleted file mode 100644
index b857d57e..00000000
--- a/src/render_helpers/shaders/resize-epilogue.frag
+++ /dev/null
@@ -1,9 +0,0 @@
-
-void main() {
-    vec3 coords_curr_geo = niri_input_to_curr_geo * vec3(niri_v_coords, 1.0);
-    vec3 size_curr_geo = vec3(niri_curr_geo_size, 1.0);
-
-    vec4 color = resize_color(coords_curr_geo, size_curr_geo);
-
-    gl_FragColor = color * niri_alpha;
-}
diff --git a/src/render_helpers/shaders/resize-prelude.frag b/src/render_helpers/shaders/resize-prelude.frag
deleted file mode 100644
index 86423ce9..00000000
--- a/src/render_helpers/shaders/resize-prelude.frag
+++ /dev/null
@@ -1,22 +0,0 @@
-#version 100
-
-precision mediump float;
-
-varying vec2 niri_v_coords;
-uniform vec2 niri_size;
-
-uniform mat3 niri_input_to_curr_geo;
-uniform mat3 niri_curr_geo_to_prev_geo;
-uniform mat3 niri_curr_geo_to_next_geo;
-uniform vec2 niri_curr_geo_size;
-
-uniform sampler2D niri_tex_prev;
-uniform mat3 niri_geo_to_tex_prev;
-
-uniform sampler2D niri_tex_next;
-uniform mat3 niri_geo_to_tex_next;
-
-uniform float niri_progress;
-uniform float niri_clamped_progress;
-
-uniform float niri_alpha;
diff --git a/src/render_helpers/shaders/resize_epilogue.frag b/src/render_helpers/shaders/resize_epilogue.frag
new file mode 100644
index 00000000..12ed890f
--- /dev/null
+++ b/src/render_helpers/shaders/resize_epilogue.frag
@@ -0,0 +1,27 @@
+
+void main() {
+    vec3 coords_curr_geo = niri_input_to_curr_geo * vec3(niri_v_coords, 1.0);
+    vec3 size_curr_geo = vec3(niri_curr_geo_size, 1.0);
+
+    vec4 color = resize_color(coords_curr_geo, size_curr_geo);
+
+    if (niri_clip_to_geometry == 1.0) {
+        if (coords_curr_geo.x < 0.0 || 1.0 < coords_curr_geo.x
+                || coords_curr_geo.y < 0.0 || 1.0 < coords_curr_geo.y) {
+            // Clip outside geometry.
+            color = vec4(0.0);
+        } else {
+            // Apply corner rounding inside geometry.
+            color = color * niri_rounding_alpha(coords_curr_geo.xy * size_curr_geo.xy, size_curr_geo.xy);
+        }
+    }
+
+    color = color * niri_alpha;
+
+#if defined(DEBUG_FLAGS)
+    if (niri_tint == 1.0)
+        color = vec4(0.0, 0.2, 0.0, 0.2) + color * 0.8;
+#endif
+
+    gl_FragColor = color;
+}
diff --git a/src/render_helpers/shaders/resize_prelude.frag b/src/render_helpers/shaders/resize_prelude.frag
new file mode 100644
index 00000000..5c052277
--- /dev/null
+++ b/src/render_helpers/shaders/resize_prelude.frag
@@ -0,0 +1,51 @@
+precision mediump float;
+
+#if defined(DEBUG_FLAGS)
+uniform float niri_tint;
+#endif
+
+varying vec2 niri_v_coords;
+uniform vec2 niri_size;
+
+uniform mat3 niri_input_to_curr_geo;
+uniform mat3 niri_curr_geo_to_prev_geo;
+uniform mat3 niri_curr_geo_to_next_geo;
+uniform vec2 niri_curr_geo_size;
+
+uniform sampler2D niri_tex_prev;
+uniform mat3 niri_geo_to_tex_prev;
+
+uniform sampler2D niri_tex_next;
+uniform mat3 niri_geo_to_tex_next;
+
+uniform float niri_progress;
+uniform float niri_clamped_progress;
+
+uniform vec4 niri_corner_radius;
+uniform float niri_clip_to_geometry;
+
+uniform float niri_alpha;
+
+float niri_rounding_alpha(vec2 coords, vec2 size) {
+    vec2 center;
+    float radius;
+
+    if (coords.x < niri_corner_radius.x && coords.y < niri_corner_radius.x) {
+        radius = niri_corner_radius.x;
+        center = vec2(radius, radius);
+    } else if (size.x - niri_corner_radius.y < coords.x && coords.y < niri_corner_radius.y) {
+        radius = niri_corner_radius.y;
+        center = vec2(size.x - radius, radius);
+    } else if (size.x - niri_corner_radius.z < coords.x && size.y - niri_corner_radius.z < coords.y) {
+        radius = niri_corner_radius.z;
+        center = vec2(size.x - radius, size.y - radius);
+    } else if (coords.x < niri_corner_radius.w && size.y - niri_corner_radius.w < coords.y) {
+        radius = niri_corner_radius.w;
+        center = vec2(radius, size.y - radius);
+    } else {
+        return 1.0;
+    }
+
+    float dist = distance(coords, center);
+    return 1.0 - smoothstep(radius - 0.5, radius + 0.5, dist);
+}