Implement layer rules: opacity and block-out-from

5 files changed, 258 insertions, 17 deletions

diff --git a/src/handlers/layer_shell.rs b/src/handlers/layer_shell.rs
index ad255c99..1888e11c 100644
--- a/src/handlers/layer_shell.rs
+++ b/src/handlers/layer_shell.rs
@@ -11,6 +11,7 @@ use smithay::wayland::shell::wlr_layer::{
 };
 use smithay::wayland::shell::xdg::PopupSurface;
 
+use crate::layer::{MappedLayer, ResolvedLayerRules};
 use crate::niri::State;
 use crate::utils::send_scale_transform;
 
@@ -60,6 +61,7 @@ impl WlrLayerShellHandler for State {
                 layer.map(|layer| (o.clone(), map, layer))
             }) {
             map.unmap_layer(&layer);
+            self.niri.mapped_layer_surfaces.remove(&layer);
             Some(output)
         } else {
             None
@@ -128,6 +130,21 @@ impl State {
                 if is_mapped {
                     let was_unmapped = self.niri.unmapped_layer_surfaces.remove(surface);
 
+                    // Resolve rules for newly mapped layer surfaces.
+                    if was_unmapped {
+                        let rules = &self.niri.config.borrow().layer_rules;
+                        let rules =
+                            ResolvedLayerRules::compute(rules, layer, self.niri.is_at_startup);
+                        let mapped = MappedLayer::new(layer.clone(), rules);
+                        let prev = self
+                            .niri
+                            .mapped_layer_surfaces
+                            .insert(layer.clone(), mapped);
+                        if prev.is_some() {
+                            error!("MappedLayer was present for an unmapped surface");
+                        }
+                    }
+
                     // Give focus to newly mapped on-demand surfaces. Some launchers like
                     // lxqt-runner rely on this behavior. While this behavior doesn't make much
                     // sense for other clients like panels, the consensus seems to be that it's not
@@ -151,6 +168,7 @@ impl State {
                         self.niri.layer_shell_on_demand_focus = Some(layer.clone());
                     }
                 } else {
+                    self.niri.mapped_layer_surfaces.remove(layer);
                     self.niri.unmapped_layer_surfaces.insert(surface.clone());
                 }
             } else {
diff --git a/src/layer/mapped.rs b/src/layer/mapped.rs
new file mode 100644
index 00000000..a70503d7
--- /dev/null
+++ b/src/layer/mapped.rs
@@ -0,0 +1,122 @@
+use std::cell::RefCell;
+
+use niri_config::layer_rule::LayerRule;
+use smithay::backend::renderer::element::surface::{
+    render_elements_from_surface_tree, WaylandSurfaceRenderElement,
+};
+use smithay::backend::renderer::element::Kind;
+use smithay::desktop::{LayerSurface, PopupManager};
+use smithay::utils::{Logical, Rectangle, Scale};
+
+use super::ResolvedLayerRules;
+use crate::niri_render_elements;
+use crate::render_helpers::renderer::NiriRenderer;
+use crate::render_helpers::solid_color::{SolidColorBuffer, SolidColorRenderElement};
+use crate::render_helpers::{RenderTarget, SplitElements};
+
+#[derive(Debug)]
+pub struct MappedLayer {
+    /// The surface itself.
+    surface: LayerSurface,
+
+    /// Up-to-date rules.
+    rules: ResolvedLayerRules,
+
+    /// Buffer to draw instead of the surface when it should be blocked out.
+    block_out_buffer: RefCell<SolidColorBuffer>,
+}
+
+niri_render_elements! {
+    LayerSurfaceRenderElement<R> => {
+        Wayland = WaylandSurfaceRenderElement<R>,
+        SolidColor = SolidColorRenderElement,
+    }
+}
+
+impl MappedLayer {
+    pub fn new(surface: LayerSurface, rules: ResolvedLayerRules) -> Self {
+        Self {
+            surface,
+            rules,
+            block_out_buffer: RefCell::new(SolidColorBuffer::new((0., 0.), [0., 0., 0., 1.])),
+        }
+    }
+
+    pub fn surface(&self) -> &LayerSurface {
+        &self.surface
+    }
+
+    pub fn rules(&self) -> &ResolvedLayerRules {
+        &self.rules
+    }
+
+    /// Recomputes the resolved layer rules and returns whether they changed.
+    pub fn recompute_layer_rules(&mut self, rules: &[LayerRule], is_at_startup: bool) -> bool {
+        let new_rules = ResolvedLayerRules::compute(rules, &self.surface, is_at_startup);
+        if new_rules == self.rules {
+            return false;
+        }
+
+        self.rules = new_rules;
+        true
+    }
+
+    pub fn render<R: NiriRenderer>(
+        &self,
+        renderer: &mut R,
+        geometry: Rectangle<i32, Logical>,
+        scale: Scale<f64>,
+        target: RenderTarget,
+    ) -> SplitElements<LayerSurfaceRenderElement<R>> {
+        let mut rv = SplitElements::default();
+
+        let alpha = self.rules.opacity.unwrap_or(1.).clamp(0., 1.);
+
+        if target.should_block_out(self.rules.block_out_from) {
+            // Round to physical pixels.
+            let geometry = geometry
+                .to_f64()
+                .to_physical_precise_round(scale)
+                .to_logical(scale);
+
+            let mut buffer = self.block_out_buffer.borrow_mut();
+            buffer.resize(geometry.size.to_f64());
+            let elem = SolidColorRenderElement::from_buffer(
+                &buffer,
+                geometry.loc,
+                alpha,
+                Kind::Unspecified,
+            );
+            rv.normal.push(elem.into());
+        } else {
+            // Layer surfaces don't have extra geometry like windows.
+            let buf_pos = geometry.loc;
+
+            let surface = self.surface.wl_surface();
+            for (popup, popup_offset) in PopupManager::popups_for_surface(surface) {
+                // Layer surfaces don't have extra geometry like windows.
+                let offset = popup_offset - popup.geometry().loc;
+
+                rv.popups.extend(render_elements_from_surface_tree(
+                    renderer,
+                    popup.wl_surface(),
+                    (buf_pos + offset).to_physical_precise_round(scale),
+                    scale,
+                    alpha,
+                    Kind::Unspecified,
+                ));
+            }
+
+            rv.normal = render_elements_from_surface_tree(
+                renderer,
+                surface,
+                buf_pos.to_physical_precise_round(scale),
+                scale,
+                alpha,
+                Kind::Unspecified,
+            );
+        }
+
+        rv
+    }
+}
diff --git a/src/layer/mod.rs b/src/layer/mod.rs
new file mode 100644
index 00000000..4d4d7819
--- /dev/null
+++ b/src/layer/mod.rs
@@ -0,0 +1,69 @@
+use niri_config::layer_rule::{LayerRule, Match};
+use niri_config::BlockOutFrom;
+use smithay::desktop::LayerSurface;
+
+pub mod mapped;
+pub use mapped::MappedLayer;
+
+/// Rules fully resolved for a layer-shell surface.
+#[derive(Debug, PartialEq)]
+pub struct ResolvedLayerRules {
+    /// Extra opacity to draw this window with.
+    pub opacity: Option<f32>,
+    /// Whether to block out this window from certain render targets.
+    pub block_out_from: Option<BlockOutFrom>,
+}
+
+impl ResolvedLayerRules {
+    pub const fn empty() -> Self {
+        Self {
+            opacity: None,
+            block_out_from: None,
+        }
+    }
+
+    pub fn compute(rules: &[LayerRule], surface: &LayerSurface, is_at_startup: bool) -> Self {
+        let _span = tracy_client::span!("ResolvedLayerRules::compute");
+
+        let mut resolved = ResolvedLayerRules::empty();
+
+        for rule in rules {
+            let matches = |m: &Match| {
+                if let Some(at_startup) = m.at_startup {
+                    if at_startup != is_at_startup {
+                        return false;
+                    }
+                }
+
+                surface_matches(surface, m)
+            };
+
+            if !(rule.matches.is_empty() || rule.matches.iter().any(matches)) {
+                continue;
+            }
+
+            if rule.excludes.iter().any(matches) {
+                continue;
+            }
+
+            if let Some(x) = rule.opacity {
+                resolved.opacity = Some(x);
+            }
+            if let Some(x) = rule.block_out_from {
+                resolved.block_out_from = Some(x);
+            }
+        }
+
+        resolved
+    }
+}
+
+fn surface_matches(surface: &LayerSurface, m: &Match) -> bool {
+    if let Some(namespace_re) = &m.namespace {
+        if !namespace_re.0.is_match(surface.namespace()) {
+            return false;
+        }
+    }
+
+    true
+}
diff --git a/src/lib.rs b/src/lib.rs
index b3ba07e1..abce64a1 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -11,6 +11,7 @@ pub mod frame_clock;
 pub mod handlers;
 pub mod input;
 pub mod ipc;
+pub mod layer;
 pub mod layout;
 pub mod niri;
 pub mod protocols;
diff --git a/src/niri.rs b/src/niri.rs
index 469d6ab8..47f92d0a 100644
--- a/src/niri.rs
+++ b/src/niri.rs
@@ -28,8 +28,8 @@ use smithay::backend::renderer::element::utils::{
     select_dmabuf_feedback, Relocate, RelocateRenderElement,
 };
 use smithay::backend::renderer::element::{
-    default_primary_scanout_output_compare, AsRenderElements, Element as _, Id, Kind,
-    PrimaryScanoutOutput, RenderElementStates,
+    default_primary_scanout_output_compare, Element as _, Id, Kind, PrimaryScanoutOutput,
+    RenderElementStates,
 };
 use smithay::backend::renderer::gles::GlesRenderer;
 use smithay::backend::renderer::sync::SyncPoint;
@@ -116,6 +116,8 @@ use crate::input::{
     apply_libinput_settings, mods_with_finger_scroll_binds, mods_with_wheel_binds, TabletData,
 };
 use crate::ipc::server::IpcServer;
+use crate::layer::mapped::LayerSurfaceRenderElement;
+use crate::layer::MappedLayer;
 use crate::layout::tile::TileRenderElement;
 use crate::layout::workspace::WorkspaceId;
 use crate::layout::{Layout, LayoutElement as _, MonitorRenderElement};
@@ -191,6 +193,9 @@ pub struct Niri {
     /// Layer surfaces which don't have a buffer attached yet.
     pub unmapped_layer_surfaces: HashSet<WlSurface>,
 
+    /// Extra data for mapped layer surfaces.
+    pub mapped_layer_surfaces: HashMap<LayerSurface, MappedLayer>,
+
     // Cached root surface for every surface, so that we can access it in destroyed() where the
     // normal get_parent() is cleared out.
     pub root_surface: HashMap<WlSurface, WlSurface>,
@@ -1012,6 +1017,7 @@ impl State {
         let mut output_config_changed = false;
         let mut preserved_output_config = None;
         let mut window_rules_changed = false;
+        let mut layer_rules_changed = false;
         let mut debug_config_changed = false;
         let mut shaders_changed = false;
         let mut cursor_inactivity_timeout_changed = false;
@@ -1071,6 +1077,10 @@ impl State {
             window_rules_changed = true;
         }
 
+        if config.layer_rules != old_config.layer_rules {
+            layer_rules_changed = true;
+        }
+
         if config.animations.window_resize.custom_shader
             != old_config.animations.window_resize.custom_shader
         {
@@ -1153,6 +1163,10 @@ impl State {
             self.niri.recompute_window_rules();
         }
 
+        if layer_rules_changed {
+            self.niri.recompute_layer_rules();
+        }
+
         if shaders_changed {
             self.niri.layout.update_shaders();
         }
@@ -1816,6 +1830,7 @@ impl Niri {
                     let _span = tracy_client::span!("startup timeout");
                     state.niri.is_at_startup = false;
                     state.niri.recompute_window_rules();
+                    state.niri.recompute_layer_rules();
                     TimeoutAction::Drop
                 },
             )
@@ -1839,6 +1854,7 @@ impl Niri {
             output_state: HashMap::new(),
             unmapped_windows: HashMap::new(),
             unmapped_layer_surfaces: HashSet::new(),
+            mapped_layer_surfaces: HashMap::new(),
             root_surface: HashMap::new(),
             dmabuf_pre_commit_hook: HashMap::new(),
             blocker_cleared_tx,
@@ -3110,7 +3126,7 @@ impl Niri {
         // Get layer-shell elements.
         let layer_map = layer_map_for_output(output);
         let mut extend_from_layer = |elements: &mut Vec<OutputRenderElements<R>>, layer| {
-            self.render_layer(renderer, output_scale, &layer_map, layer, elements);
+            self.render_layer(renderer, target, output_scale, &layer_map, layer, elements);
         };
 
         // The upper layer-shell elements go next.
@@ -3144,7 +3160,8 @@ impl Niri {
     fn render_layer<R: NiriRenderer>(
         &self,
         renderer: &mut R,
-        output_scale: Scale<f64>,
+        target: RenderTarget,
+        scale: Scale<f64>,
         layer_map: &LayerMap,
         layer: Layer,
         elements: &mut Vec<OutputRenderElements<R>>,
@@ -3152,20 +3169,13 @@ impl Niri {
         let iter = layer_map
             .layers_on(layer)
             .filter_map(|surface| {
-                layer_map
-                    .layer_geometry(surface)
-                    .map(|geo| (geo.loc, surface))
+                let mapped = self.mapped_layer_surfaces.get(surface)?;
+                let geo = layer_map.layer_geometry(surface)?;
+                Some((mapped, geo))
             })
-            .flat_map(|(loc, surface)| {
-                surface
-                    .render_elements(
-                        renderer,
-                        loc.to_physical_precise_round(output_scale),
-                        output_scale,
-                        1.,
-                    )
-                    .into_iter()
-                    .map(OutputRenderElements::Wayland)
+            .flat_map(|(mapped, geo)| {
+                let elements = mapped.render(renderer, geo, scale, target);
+                elements.into_iter().map(OutputRenderElements::LayerSurface)
             });
         elements.extend(iter);
     }
@@ -4805,6 +4815,26 @@ impl Niri {
         }
     }
 
+    pub fn recompute_layer_rules(&mut self) {
+        let _span = tracy_client::span!("Niri::recompute_layer_rules");
+
+        let mut changed = false;
+        {
+            let rules = &self.config.borrow().layer_rules;
+
+            for mapped in self.mapped_layer_surfaces.values_mut() {
+                if mapped.recompute_layer_rules(rules, self.is_at_startup) {
+                    changed = true;
+                }
+            }
+        }
+
+        if changed {
+            // FIXME: granular.
+            self.queue_redraw_all();
+        }
+    }
+
     pub fn reset_pointer_inactivity_timer(&mut self) {
         let _span = tracy_client::span!("Niri::reset_pointer_inactivity_timer");
 
@@ -4850,6 +4880,7 @@ niri_render_elements! {
     OutputRenderElements<R> => {
         Monitor = MonitorRenderElement<R>,
         Tile = TileRenderElement<R>,
+        LayerSurface = LayerSurfaceRenderElement<R>,
         Wayland = WaylandSurfaceRenderElement<R>,
         NamedPointer = MemoryRenderBufferRenderElement<R>,
         SolidColor = SolidColorRenderElement,