Implement layer rules: opacity and block-out-from

11 files changed, 401 insertions, 17 deletions

diff --git a/niri-config/src/layer_rule.rs b/niri-config/src/layer_rule.rs
new file mode 100644
index 00000000..dc6fbd8d
--- /dev/null
+++ b/niri-config/src/layer_rule.rs
@@ -0,0 +1,22 @@
+use crate::{BlockOutFrom, RegexEq};
+
+#[derive(knuffel::Decode, Debug, Default, Clone, PartialEq)]
+pub struct LayerRule {
+    #[knuffel(children(name = "match"))]
+    pub matches: Vec<Match>,
+    #[knuffel(children(name = "exclude"))]
+    pub excludes: Vec<Match>,
+
+    #[knuffel(child, unwrap(argument))]
+    pub opacity: Option<f32>,
+    #[knuffel(child, unwrap(argument))]
+    pub block_out_from: Option<BlockOutFrom>,
+}
+
+#[derive(knuffel::Decode, Debug, Default, Clone, PartialEq)]
+pub struct Match {
+    #[knuffel(property, str)]
+    pub namespace: Option<RegexEq>,
+    #[knuffel(property)]
+    pub at_startup: Option<bool>,
+}
diff --git a/niri-config/src/lib.rs b/niri-config/src/lib.rs
index 7fa4f527..5c5f2cf7 100644
--- a/niri-config/src/lib.rs
+++ b/niri-config/src/lib.rs
@@ -10,6 +10,7 @@ use std::time::Duration;
 use bitflags::bitflags;
 use knuffel::errors::DecodeError;
 use knuffel::Decode as _;
+use layer_rule::LayerRule;
 use miette::{miette, Context, IntoDiagnostic, NarratableReportHandler};
 use niri_ipc::{ConfiguredMode, LayoutSwitchTarget, SizeChange, Transform, WorkspaceReferenceArg};
 use smithay::backend::renderer::Color32F;
@@ -20,6 +21,8 @@ use smithay::reexports::input;
 
 pub const DEFAULT_BACKGROUND_COLOR: Color = Color::from_array_unpremul([0.2, 0.2, 0.2, 1.]);
 
+pub mod layer_rule;
+
 mod utils;
 pub use utils::RegexEq;
 
@@ -53,6 +56,8 @@ pub struct Config {
     pub environment: Environment,
     #[knuffel(children(name = "window-rule"))]
     pub window_rules: Vec<WindowRule>,
+    #[knuffel(children(name = "layer-rule"))]
+    pub layer_rules: Vec<LayerRule>,
     #[knuffel(child, default)]
     pub binds: Binds,
     #[knuffel(child, default)]
@@ -3119,6 +3124,11 @@ mod tests {
                 }
             }
 
+            layer-rule {
+                match namespace="^notifications$"
+                block-out-from "screencast"
+            }
+
             binds {
                 Mod+T allow-when-locked=true { spawn "alacritty"; }
                 Mod+Q { close-window; }
@@ -3391,6 +3401,17 @@ mod tests {
                     },
                     ..Default::default()
                 }],
+                layer_rules: vec![
+                    LayerRule {
+                        matches: vec![layer_rule::Match {
+                            namespace: Some(RegexEq::from_str("^notifications$").unwrap()),
+                            at_startup: None,
+                        }],
+                        excludes: vec![],
+                        opacity: None,
+                        block_out_from: Some(BlockOutFrom::Screencast),
+                    }
+                ],
                 workspaces: vec![
                     Workspace {
                         name: WorkspaceName("workspace-1".to_string()),
diff --git a/src/handlers/layer_shell.rs b/src/handlers/layer_shell.rs
index ad255c99..1888e11c 100644
--- a/src/handlers/layer_shell.rs
+++ b/src/handlers/layer_shell.rs
@@ -11,6 +11,7 @@ use smithay::wayland::shell::wlr_layer::{
 };
 use smithay::wayland::shell::xdg::PopupSurface;
 
+use crate::layer::{MappedLayer, ResolvedLayerRules};
 use crate::niri::State;
 use crate::utils::send_scale_transform;
 
@@ -60,6 +61,7 @@ impl WlrLayerShellHandler for State {
                 layer.map(|layer| (o.clone(), map, layer))
             }) {
             map.unmap_layer(&layer);
+            self.niri.mapped_layer_surfaces.remove(&layer);
             Some(output)
         } else {
             None
@@ -128,6 +130,21 @@ impl State {
                 if is_mapped {
                     let was_unmapped = self.niri.unmapped_layer_surfaces.remove(surface);
 
+                    // Resolve rules for newly mapped layer surfaces.
+                    if was_unmapped {
+                        let rules = &self.niri.config.borrow().layer_rules;
+                        let rules =
+                            ResolvedLayerRules::compute(rules, layer, self.niri.is_at_startup);
+                        let mapped = MappedLayer::new(layer.clone(), rules);
+                        let prev = self
+                            .niri
+                            .mapped_layer_surfaces
+                            .insert(layer.clone(), mapped);
+                        if prev.is_some() {
+                            error!("MappedLayer was present for an unmapped surface");
+                        }
+                    }
+
                     // Give focus to newly mapped on-demand surfaces. Some launchers like
                     // lxqt-runner rely on this behavior. While this behavior doesn't make much
                     // sense for other clients like panels, the consensus seems to be that it's not
@@ -151,6 +168,7 @@ impl State {
                         self.niri.layer_shell_on_demand_focus = Some(layer.clone());
                     }
                 } else {
+                    self.niri.mapped_layer_surfaces.remove(layer);
                     self.niri.unmapped_layer_surfaces.insert(surface.clone());
                 }
             } else {
diff --git a/src/layer/mapped.rs b/src/layer/mapped.rs
new file mode 100644
index 00000000..a70503d7
--- /dev/null
+++ b/src/layer/mapped.rs
@@ -0,0 +1,122 @@
+use std::cell::RefCell;
+
+use niri_config::layer_rule::LayerRule;
+use smithay::backend::renderer::element::surface::{
+    render_elements_from_surface_tree, WaylandSurfaceRenderElement,
+};
+use smithay::backend::renderer::element::Kind;
+use smithay::desktop::{LayerSurface, PopupManager};
+use smithay::utils::{Logical, Rectangle, Scale};
+
+use super::ResolvedLayerRules;
+use crate::niri_render_elements;
+use crate::render_helpers::renderer::NiriRenderer;
+use crate::render_helpers::solid_color::{SolidColorBuffer, SolidColorRenderElement};
+use crate::render_helpers::{RenderTarget, SplitElements};
+
+#[derive(Debug)]
+pub struct MappedLayer {
+    /// The surface itself.
+    surface: LayerSurface,
+
+    /// Up-to-date rules.
+    rules: ResolvedLayerRules,
+
+    /// Buffer to draw instead of the surface when it should be blocked out.
+    block_out_buffer: RefCell<SolidColorBuffer>,
+}
+
+niri_render_elements! {
+    LayerSurfaceRenderElement<R> => {
+        Wayland = WaylandSurfaceRenderElement<R>,
+        SolidColor = SolidColorRenderElement,
+    }
+}
+
+impl MappedLayer {
+    pub fn new(surface: LayerSurface, rules: ResolvedLayerRules) -> Self {
+        Self {
+            surface,
+            rules,
+            block_out_buffer: RefCell::new(SolidColorBuffer::new((0., 0.), [0., 0., 0., 1.])),
+        }
+    }
+
+    pub fn surface(&self) -> &LayerSurface {
+        &self.surface
+    }
+
+    pub fn rules(&self) -> &ResolvedLayerRules {
+        &self.rules
+    }
+
+    /// Recomputes the resolved layer rules and returns whether they changed.
+    pub fn recompute_layer_rules(&mut self, rules: &[LayerRule], is_at_startup: bool) -> bool {
+        let new_rules = ResolvedLayerRules::compute(rules, &self.surface, is_at_startup);
+        if new_rules == self.rules {
+            return false;
+        }
+
+        self.rules = new_rules;
+        true
+    }
+
+    pub fn render<R: NiriRenderer>(
+        &self,
+        renderer: &mut R,
+        geometry: Rectangle<i32, Logical>,
+        scale: Scale<f64>,
+        target: RenderTarget,
+    ) -> SplitElements<LayerSurfaceRenderElement<R>> {
+        let mut rv = SplitElements::default();
+
+        let alpha = self.rules.opacity.unwrap_or(1.).clamp(0., 1.);
+
+        if target.should_block_out(self.rules.block_out_from) {
+            // Round to physical pixels.
+            let geometry = geometry
+                .to_f64()
+                .to_physical_precise_round(scale)
+                .to_logical(scale);
+
+            let mut buffer = self.block_out_buffer.borrow_mut();
+            buffer.resize(geometry.size.to_f64());
+            let elem = SolidColorRenderElement::from_buffer(
+                &buffer,
+                geometry.loc,
+                alpha,
+                Kind::Unspecified,
+            );
+            rv.normal.push(elem.into());
+        } else {
+            // Layer surfaces don't have extra geometry like windows.
+            let buf_pos = geometry.loc;
+
+            let surface = self.surface.wl_surface();
+            for (popup, popup_offset) in PopupManager::popups_for_surface(surface) {
+                // Layer surfaces don't have extra geometry like windows.
+                let offset = popup_offset - popup.geometry().loc;
+
+                rv.popups.extend(render_elements_from_surface_tree(
+                    renderer,
+                    popup.wl_surface(),
+                    (buf_pos + offset).to_physical_precise_round(scale),
+                    scale,
+                    alpha,
+                    Kind::Unspecified,
+                ));
+            }
+
+            rv.normal = render_elements_from_surface_tree(
+                renderer,
+                surface,
+                buf_pos.to_physical_precise_round(scale),
+                scale,
+                alpha,
+                Kind::Unspecified,
+            );
+        }
+
+        rv
+    }
+}
diff --git a/src/layer/mod.rs b/src/layer/mod.rs
new file mode 100644
index 00000000..4d4d7819
--- /dev/null
+++ b/src/layer/mod.rs
@@ -0,0 +1,69 @@
+use niri_config::layer_rule::{LayerRule, Match};
+use niri_config::BlockOutFrom;
+use smithay::desktop::LayerSurface;
+
+pub mod mapped;
+pub use mapped::MappedLayer;
+
+/// Rules fully resolved for a layer-shell surface.
+#[derive(Debug, PartialEq)]
+pub struct ResolvedLayerRules {
+    /// Extra opacity to draw this window with.
+    pub opacity: Option<f32>,
+    /// Whether to block out this window from certain render targets.
+    pub block_out_from: Option<BlockOutFrom>,
+}
+
+impl ResolvedLayerRules {
+    pub const fn empty() -> Self {
+        Self {
+            opacity: None,
+            block_out_from: None,
+        }
+    }
+
+    pub fn compute(rules: &[LayerRule], surface: &LayerSurface, is_at_startup: bool) -> Self {
+        let _span = tracy_client::span!("ResolvedLayerRules::compute");
+
+        let mut resolved = ResolvedLayerRules::empty();
+
+        for rule in rules {
+            let matches = |m: &Match| {
+                if let Some(at_startup) = m.at_startup {
+                    if at_startup != is_at_startup {
+                        return false;
+                    }
+                }
+
+                surface_matches(surface, m)
+            };
+
+            if !(rule.matches.is_empty() || rule.matches.iter().any(matches)) {
+                continue;
+            }
+
+            if rule.excludes.iter().any(matches) {
+                continue;
+            }
+
+            if let Some(x) = rule.opacity {
+                resolved.opacity = Some(x);
+            }
+            if let Some(x) = rule.block_out_from {
+                resolved.block_out_from = Some(x);
+            }
+        }
+
+        resolved
+    }
+}
+
+fn surface_matches(surface: &LayerSurface, m: &Match) -> bool {
+    if let Some(namespace_re) = &m.namespace {
+        if !namespace_re.0.is_match(surface.namespace()) {
+            return false;
+        }
+    }
+
+    true
+}
diff --git a/src/lib.rs b/src/lib.rs
index b3ba07e1..abce64a1 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -11,6 +11,7 @@ pub mod frame_clock;
 pub mod handlers;
 pub mod input;
 pub mod ipc;
+pub mod layer;
 pub mod layout;
 pub mod niri;
 pub mod protocols;
diff --git a/src/niri.rs b/src/niri.rs
index 469d6ab8..47f92d0a 100644
--- a/src/niri.rs
+++ b/src/niri.rs
@@ -28,8 +28,8 @@ use smithay::backend::renderer::element::utils::{
     select_dmabuf_feedback, Relocate, RelocateRenderElement,
 };
 use smithay::backend::renderer::element::{
-    default_primary_scanout_output_compare, AsRenderElements, Element as _, Id, Kind,
-    PrimaryScanoutOutput, RenderElementStates,
+    default_primary_scanout_output_compare, Element as _, Id, Kind, PrimaryScanoutOutput,
+    RenderElementStates,
 };
 use smithay::backend::renderer::gles::GlesRenderer;
 use smithay::backend::renderer::sync::SyncPoint;
@@ -116,6 +116,8 @@ use crate::input::{
     apply_libinput_settings, mods_with_finger_scroll_binds, mods_with_wheel_binds, TabletData,
 };
 use crate::ipc::server::IpcServer;
+use crate::layer::mapped::LayerSurfaceRenderElement;
+use crate::layer::MappedLayer;
 use crate::layout::tile::TileRenderElement;
 use crate::layout::workspace::WorkspaceId;
 use crate::layout::{Layout, LayoutElement as _, MonitorRenderElement};
@@ -191,6 +193,9 @@ pub struct Niri {
     /// Layer surfaces which don't have a buffer attached yet.
     pub unmapped_layer_surfaces: HashSet<WlSurface>,
 
+    /// Extra data for mapped layer surfaces.
+    pub mapped_layer_surfaces: HashMap<LayerSurface, MappedLayer>,
+
     // Cached root surface for every surface, so that we can access it in destroyed() where the
     // normal get_parent() is cleared out.
     pub root_surface: HashMap<WlSurface, WlSurface>,
@@ -1012,6 +1017,7 @@ impl State {
         let mut output_config_changed = false;
         let mut preserved_output_config = None;
         let mut window_rules_changed = false;
+        let mut layer_rules_changed = false;
         let mut debug_config_changed = false;
         let mut shaders_changed = false;
         let mut cursor_inactivity_timeout_changed = false;
@@ -1071,6 +1077,10 @@ impl State {
             window_rules_changed = true;
         }
 
+        if config.layer_rules != old_config.layer_rules {
+            layer_rules_changed = true;
+        }
+
         if config.animations.window_resize.custom_shader
             != old_config.animations.window_resize.custom_shader
         {
@@ -1153,6 +1163,10 @@ impl State {
             self.niri.recompute_window_rules();
         }
 
+        if layer_rules_changed {
+            self.niri.recompute_layer_rules();
+        }
+
         if shaders_changed {
             self.niri.layout.update_shaders();
         }
@@ -1816,6 +1830,7 @@ impl Niri {
                     let _span = tracy_client::span!("startup timeout");
                     state.niri.is_at_startup = false;
                     state.niri.recompute_window_rules();
+                    state.niri.recompute_layer_rules();
                     TimeoutAction::Drop
                 },
             )
@@ -1839,6 +1854,7 @@ impl Niri {
             output_state: HashMap::new(),
             unmapped_windows: HashMap::new(),
             unmapped_layer_surfaces: HashSet::new(),
+            mapped_layer_surfaces: HashMap::new(),
             root_surface: HashMap::new(),
             dmabuf_pre_commit_hook: HashMap::new(),
             blocker_cleared_tx,
@@ -3110,7 +3126,7 @@ impl Niri {
         // Get layer-shell elements.
         let layer_map = layer_map_for_output(output);
         let mut extend_from_layer = |elements: &mut Vec<OutputRenderElements<R>>, layer| {
-            self.render_layer(renderer, output_scale, &layer_map, layer, elements);
+            self.render_layer(renderer, target, output_scale, &layer_map, layer, elements);
         };
 
         // The upper layer-shell elements go next.
@@ -3144,7 +3160,8 @@ impl Niri {
     fn render_layer<R: NiriRenderer>(
         &self,
         renderer: &mut R,
-        output_scale: Scale<f64>,
+        target: RenderTarget,
+        scale: Scale<f64>,
         layer_map: &LayerMap,
         layer: Layer,
         elements: &mut Vec<OutputRenderElements<R>>,
@@ -3152,20 +3169,13 @@ impl Niri {
         let iter = layer_map
             .layers_on(layer)
             .filter_map(|surface| {
-                layer_map
-                    .layer_geometry(surface)
-                    .map(|geo| (geo.loc, surface))
+                let mapped = self.mapped_layer_surfaces.get(surface)?;
+                let geo = layer_map.layer_geometry(surface)?;
+                Some((mapped, geo))
             })
-            .flat_map(|(loc, surface)| {
-                surface
-                    .render_elements(
-                        renderer,
-                        loc.to_physical_precise_round(output_scale),
-                        output_scale,
-                        1.,
-                    )
-                    .into_iter()
-                    .map(OutputRenderElements::Wayland)
+            .flat_map(|(mapped, geo)| {
+                let elements = mapped.render(renderer, geo, scale, target);
+                elements.into_iter().map(OutputRenderElements::LayerSurface)
             });
         elements.extend(iter);
     }
@@ -4805,6 +4815,26 @@ impl Niri {
         }
     }
 
+    pub fn recompute_layer_rules(&mut self) {
+        let _span = tracy_client::span!("Niri::recompute_layer_rules");
+
+        let mut changed = false;
+        {
+            let rules = &self.config.borrow().layer_rules;
+
+            for mapped in self.mapped_layer_surfaces.values_mut() {
+                if mapped.recompute_layer_rules(rules, self.is_at_startup) {
+                    changed = true;
+                }
+            }
+        }
+
+        if changed {
+            // FIXME: granular.
+            self.queue_redraw_all();
+        }
+    }
+
     pub fn reset_pointer_inactivity_timer(&mut self) {
         let _span = tracy_client::span!("Niri::reset_pointer_inactivity_timer");
 
@@ -4850,6 +4880,7 @@ niri_render_elements! {
     OutputRenderElements<R> => {
         Monitor = MonitorRenderElement<R>,
         Tile = TileRenderElement<R>,
+        LayerSurface = LayerSurfaceRenderElement<R>,
         Wayland = WaylandSurfaceRenderElement<R>,
         NamedPointer = MemoryRenderBufferRenderElement<R>,
         SolidColor = SolidColorRenderElement,
diff --git a/wiki/Configuration:-Layer-Rules.md b/wiki/Configuration:-Layer-Rules.md
new file mode 100644
index 00000000..dfb50ca6
--- /dev/null
+++ b/wiki/Configuration:-Layer-Rules.md
@@ -0,0 +1,95 @@
+### Overview
+
+Layer rules let you adjust behavior for individual layer-shell surfaces.
+They have `match` and `exclude` directives that control which layer-shell surfaces the rule should apply to, and a number of properties that you can set.
+
+Layer rules are processed and work very similarly to window rules, just with different matchers and properties.
+Please read the [window rules](./Configuration:-Window-Rules.md) wiki page to learn how matching works.
+
+Here are all matchers and properties that a layer rule could have:
+
+```kdl
+layer-rule {
+    match namespace="waybar"
+    match at-startup=true
+
+    // Properties that apply continuously.
+    opacity 0.5
+    block-out-from "screencast"
+    // block-out-from "screen-capture"
+}
+```
+
+### Layer Surface Matching
+
+Let's look at the matchers in more detail.
+
+#### `namespace`
+
+This is a regular expression that should match anywhere in the surface namespace.
+You can read about the supported regular expression syntax [here](https://docs.rs/regex/latest/regex/#syntax).
+
+```kdl
+// Match surfaces with namespace containing "waybar",
+layer-rule {
+    match namespace="waybar"
+}
+```
+
+You can find the namespaces of all open layer-shell surfaces by running `niri msg layers`.
+
+#### `at-startup`
+
+Can be `true` or `false`.
+Matches during the first 60 seconds after starting niri.
+
+```kdl
+// Show layer-shell surfaces with 0.5 opacity at niri startup, but not afterwards.
+layer-rule {
+    match at-startup=true
+
+    opacity 0.5
+}
+```
+
+### Dynamic Properties
+
+These properties apply continuously to open layer-shell surfaces.
+
+#### `block-out-from`
+
+You can block out surfaces from xdg-desktop-portal screencasts or all screen captures.
+They will be replaced with solid black rectangles.
+
+This can be useful for notifications.
+
+The same caveats and instructions apply as for the `block-out-from` window rule.
+Please read the `block-out-from` section in the [window rules](./Configuration:-Window-Rules.md) wiki page for more details.
+
+![Screenshot showing a notification visible normally, but blocked out on OBS.](./img/layer-block-out-from-screencast.png)
+
+```kdl
+// Block out mako notifications from screencasts.
+layer-rule {
+    match namespace="^notifications$"
+
+    block-out-from "screencast"
+}
+```
+
+#### `opacity`
+
+Set the opacity of the surface.
+`0.0` is fully transparent, `1.0` is fully opaque.
+This is applied on top of the surface's own opacity, so semitransparent surfaces will become even more transparent.
+
+Opacity is applied to every child of the layer-shell surface individually, so subsurfaces and pop-up menus will show window content behind them.
+
+```kdl
+// Make fuzzel semitransparent.
+layer-rule {
+    match namespace="^launcher$"
+
+    opacity 0.95
+}
+```
diff --git a/wiki/Configuration:-Overview.md b/wiki/Configuration:-Overview.md
index cf208504..a0efb9cb 100644
--- a/wiki/Configuration:-Overview.md
+++ b/wiki/Configuration:-Overview.md
@@ -9,6 +9,7 @@ You can find documentation for various sections of the config on these wiki page
 * [`layout {}`](./Configuration:-Layout.md)
 * [top-level options](./Configuration:-Miscellaneous.md)
 * [`window-rule {}`](./Configuration:-Window-Rules.md)
+* [`layer-rule {}`](./Configuration:-Layer-Rules.md)
 * [`animations {}`](./Configuration:-Animations.md)
 * [`debug {}`](./Configuration:-Debug-Options.md)
 
diff --git a/wiki/_Sidebar.md b/wiki/_Sidebar.md
index dfa48e72..7ef11a66 100644
--- a/wiki/_Sidebar.md
+++ b/wiki/_Sidebar.md
@@ -19,6 +19,7 @@
 * [Named Workspaces](./Configuration:-Named-Workspaces.md)
 * [Miscellaneous](./Configuration:-Miscellaneous.md)
 * [Window Rules](./Configuration:-Window-Rules.md)
+* [Layer Rules](./Configuration:-Layer-Rules.md)
 * [Animations](./Configuration:-Animations.md)
 * [Debug Options](./Configuration:-Debug-Options.md)
 
diff --git a/wiki/img/layer-block-out-from-screencast.png b/wiki/img/layer-block-out-from-screencast.png
new file mode 100644
index 00000000..4456536f
--- /dev/null
+++ b/wiki/img/layer-block-out-from-screencast.png
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1dcaa6ece8e8287081332604270fa17a66561e0d81fd190d665005b6359c0eac
+size 559823