Clipped region fixes

include/render/fx_renderer/shaders.h

@@ -25,6 +25,14 @@ struct quad_shader {
 	GLint proj;
 	GLint color;
 	GLint pos_attrib;
+
+	GLint clip_size;
+	GLint clip_position;
+	GLint clip_corner_radius;
+	GLint clip_round_top_left;
+	GLint clip_round_top_right;
+	GLint clip_round_bottom_left;
+	GLint clip_round_bottom_right;
 };
 
 bool link_quad_program(struct quad_shader *shader);

include/scenefx/render/pass.h

@@ -60,7 +60,7 @@ struct fx_render_texture_options {
 
 struct fx_render_rect_options {
 	struct wlr_render_rect_options base;
-	// TODO: Add effects here in the future
+	struct clipped_region clipped_region;
 };
 
 struct fx_render_rect_grad_options {
@@ -72,7 +72,6 @@ struct fx_render_rounded_rect_options {
 	struct wlr_render_rect_options base;
 	int corner_radius;
 	enum corner_location corners;
-
 	struct clipped_region clipped_region;
 };
 

include/scenefx/types/wlr_scene.h

@@ -430,6 +430,8 @@ void wlr_scene_rect_set_corner_radius(struct wlr_scene_rect *rect, int corner_ra
  *
  * For there to be corner rounding of the clipped region, the corner radius and
  * corners must be non-zero.
+ *
+ * NOTE: The positioning is node-relative.
  */
 void wlr_scene_rect_set_clipped_region(struct wlr_scene_rect *rect,
 		struct clipped_region clipped_region);
@@ -483,6 +485,8 @@ void wlr_scene_shadow_set_color(struct wlr_scene_shadow *shadow, const float col
  *
  * For there to be corner rounding of the clipped region, the corner radius and
  * corners must be non-zero.
+ *
+ * NOTE: The positioning is node-relative.
  */
 void wlr_scene_shadow_set_clipped_region(struct wlr_scene_shadow *shadow,
 		struct clipped_region clipped_region);

render/fx_renderer/fx_pass.c

@@ -37,6 +37,11 @@ struct fx_render_rect_options fx_render_rect_options_default(
 		const struct wlr_render_rect_options *base) {
 	struct fx_render_rect_options options = {
 		.base = *base,
+		.clipped_region = {
+			.area = { .0, .0, .0, .0 },
+			.corner_radius = 0,
+			.corners = CORNER_LOCATION_NONE,
+		},
 	};
 	return options;
 }
@@ -345,13 +350,51 @@ void fx_render_pass_add_rect(struct fx_gles_render_pass *pass,
 	struct wlr_box box;
 	wlr_render_rect_options_get_box(options, pass->buffer->buffer, &box);
 
-	push_fx_debug(renderer);
-	setup_blending(color->a == 1.0 ? WLR_RENDER_BLEND_MODE_NONE : options->blend_mode);
+	pixman_region32_t clip_region;
+	if (options->clip) {
+		pixman_region32_init(&clip_region);
+		pixman_region32_copy(&clip_region, options->clip);
+	} else {
+		pixman_region32_init_rect(&clip_region, box.x, box.y, box.width, box.height);
+	}
+	const struct wlr_box clipped_region_box = fx_options->clipped_region.area;
+	enum corner_location clipped_region_corners = fx_options->clipped_region.corners;
+	int clipped_region_corner_radius = clipped_region_corners != CORNER_LOCATION_NONE ?
+		fx_options->clipped_region.corner_radius : 0;
+	if (!wlr_box_empty(&clipped_region_box)) {
+		pixman_region32_t user_clip_region;
+		pixman_region32_init_rect(
+			&user_clip_region,
+			clipped_region_box.x + clipped_region_corner_radius * 0.3,
+			clipped_region_box.y + clipped_region_corner_radius * 0.3,
+			fmax(clipped_region_box.width - clipped_region_corner_radius * 0.6, 0),
+			fmax(clipped_region_box.height - clipped_region_corner_radius * 0.6, 0)
+		);
+		pixman_region32_subtract(&clip_region, &clip_region, &user_clip_region);
+		pixman_region32_fini(&user_clip_region);
 
-	glUseProgram(renderer->shaders.quad.program);
+		push_fx_debug(renderer);
+		setup_blending(options->blend_mode);
+	} else {
+		push_fx_debug(renderer);
+		setup_blending(color->a == 1.0 ? WLR_RENDER_BLEND_MODE_NONE : options->blend_mode);
+	}
 
-	set_proj_matrix(renderer->shaders.quad.proj, pass->projection_matrix, &box);
-	glUniform4f(renderer->shaders.quad.color, color->r, color->g, color->b, color->a);
+	struct quad_shader shader = renderer->shaders.quad;
+	glUseProgram(shader.program);
+	set_proj_matrix(shader.proj, pass->projection_matrix, &box);
+	glUniform4f(shader.color, color->r, color->g, color->b, color->a);
+	glUniform2f(shader.clip_size, clipped_region_box.width, clipped_region_box.height);
+	glUniform2f(shader.clip_position, clipped_region_box.x, clipped_region_box.y);
+	glUniform1f(shader.clip_corner_radius, fx_options->clipped_region.corner_radius);
+	glUniform1f(shader.clip_round_top_left,
+			(CORNER_LOCATION_TOP_LEFT & clipped_region_corners) == CORNER_LOCATION_TOP_LEFT);
+	glUniform1f(shader.clip_round_top_right,
+			(CORNER_LOCATION_TOP_RIGHT & clipped_region_corners) == CORNER_LOCATION_TOP_RIGHT);
+	glUniform1f(shader.clip_round_bottom_left,
+			(CORNER_LOCATION_BOTTOM_LEFT & clipped_region_corners) == CORNER_LOCATION_BOTTOM_LEFT);
+	glUniform1f(shader.clip_round_bottom_right,
+			(CORNER_LOCATION_BOTTOM_RIGHT & clipped_region_corners) == CORNER_LOCATION_BOTTOM_RIGHT);
 
 	render(&box, options->clip, renderer->shaders.quad.pos_attrib);
 

render/fx_renderer/gles2/shaders/quad.frag

@@ -6,8 +6,28 @@ precision mediump float;
 
 varying vec4 v_color;
 varying vec2 v_texcoord;
-uniform vec4 color;
+
+uniform vec2 clip_size;
+uniform vec2 clip_position;
+uniform float clip_corner_radius;
+uniform bool clip_round_top_left;
+uniform bool clip_round_top_right;
+uniform bool clip_round_bottom_left;
+uniform bool clip_round_bottom_right;
+
+float corner_alpha(vec2 size, vec2 position, float radius,
+            bool round_tl, bool round_tr, bool round_bl, bool round_br);
 
 void main() {
-	gl_FragColor = color;
+    // Clipping
+    float clip_corner_alpha = corner_alpha(clip_size, clip_position, clip_corner_radius,
+            clip_round_top_left, clip_round_top_right,
+            clip_round_bottom_left, clip_round_bottom_right);
+
+    float base_case = float(clip_round_top_left) + float(clip_round_top_right)
+            + float(clip_round_bottom_left) + float(clip_round_bottom_right);
+    base_case *= step(1.0, clip_corner_radius); // Corner radius is 0
+    clip_corner_alpha = max(clip_corner_alpha, 1.0 - step(1.0, base_case));
+
+    gl_FragColor = v_color * clip_corner_alpha;
 }

render/fx_renderer/shaders.c

@@ -104,15 +104,26 @@ void load_gl_proc(void *proc_ptr, const char *name) {
 // Shaders
 
 bool link_quad_program(struct quad_shader *shader) {
+	GLchar quad_src[4096];
+	snprintf(quad_src, sizeof(quad_src), "%s\n%s", quad_frag_src,
+		corner_alpha_frag_src);
+
 	GLuint prog;
-	shader->program = prog = link_program(quad_frag_src);
+	shader->program = prog = link_program(quad_src);
 	if (!shader->program) {
 		return false;
 	}
 
 	shader->proj = glGetUniformLocation(prog, "proj");
 	shader->color = glGetUniformLocation(prog, "color");
 	shader->pos_attrib = glGetAttribLocation(prog, "pos");
+	shader->clip_size = glGetUniformLocation(prog, "clip_size");
+	shader->clip_position = glGetUniformLocation(prog, "clip_position");
+	shader->clip_corner_radius = glGetUniformLocation(prog, "clip_corner_radius");
+	shader->clip_round_top_left = glGetUniformLocation(prog, "clip_round_top_left");
+	shader->clip_round_top_right = glGetUniformLocation(prog, "clip_round_top_right");
+	shader->clip_round_bottom_left = glGetUniformLocation(prog, "clip_round_bottom_left");
+	shader->clip_round_bottom_right = glGetUniformLocation(prog, "clip_round_bottom_right");
 
 	return true;
 }

tinywl/tinywl.c

@@ -612,7 +612,7 @@ static void xdg_toplevel_commit(struct wl_listener *listener, void *data) {
 	wlr_scene_rect_set_clipped_region(toplevel->border, (struct clipped_region) {
 			.corner_radius = toplevel->corner_radius,
 			.corners = CORNER_LOCATION_ALL,
-			.area = { 0, 0, geometry.width, geometry.height }
+			.area = { BORDER_THICKNESS, BORDER_THICKNESS, geometry.width, geometry.height }
 	});
 
 	int blur_sigma = toplevel->shadow->blur_sigma;
@@ -622,7 +622,7 @@ static void xdg_toplevel_commit(struct wl_listener *listener, void *data) {
 	wlr_scene_shadow_set_clipped_region(toplevel->shadow, (struct clipped_region) {
 			.corner_radius = toplevel->corner_radius + BORDER_THICKNESS,
 			.corners = CORNER_LOCATION_ALL,
-			.area = { -BORDER_THICKNESS, -BORDER_THICKNESS, border_width, border_height }
+			.area = { blur_sigma, blur_sigma, border_width, border_height }
 	});
 }
 
@@ -970,7 +970,7 @@ static void server_new_xdg_toplevel(struct wl_listener *listener, void *data) {
 	toplevel->border = wlr_scene_rect_create(toplevel->scene_tree, 0, 0,
 			(float[4]){ 1.0f, 0.f, 0.f, 1.0f });
 	wlr_scene_rect_set_corner_radius(toplevel->border,
-			toplevel->corner_radius + BORDER_THICKNESS, CORNER_LOCATION_ALL);
+			toplevel->corner_radius + BORDER_THICKNESS, CORNER_LOCATION_BOTTOM);
 	wlr_scene_node_set_position(&toplevel->border->node, -BORDER_THICKNESS, -BORDER_THICKNESS);
 
 	float blur_sigma = 20.0f;
@@ -1152,6 +1152,16 @@ int main(int argc, char *argv[]) {
 	float top_rect_color[4] = { 1, 0, 0, 1 };
 	struct wlr_scene_rect *rect = wlr_scene_rect_create(server.layers.toplevel_layer,
 			200, 200, top_rect_color);
+	wlr_scene_rect_set_clipped_region(rect, (struct clipped_region) {
+			.corner_radius = 12,
+			.corners = CORNER_LOCATION_TOP,
+			.area = {
+				.x = 50,
+				.y = 50,
+				.width = 100,
+				.height = 100,
+			},
+	});
 	wlr_scene_node_set_position(&rect->node, 200, 200);
 
 	// blur

types/scene/wlr_scene.c

@@ -1602,6 +1602,18 @@ static void scene_entry_render(struct render_list_entry *entry, const struct ren
 			pixman_region32_fini(&opaque_region);
 		}
 
+		struct wlr_box rect_clipped_region_box = scene_rect->clipped_region.area;
+		int rect_clipped_region_corner_radius = scene_rect->clipped_region.corner_radius;
+		enum corner_location rect_clipped_corners = scene_rect->clipped_region.corners;
+
+		// Node relative -> Root relative
+		rect_clipped_region_box.x += x;
+		rect_clipped_region_box.y += y;
+
+		scale_box(&rect_clipped_region_box, data->scale);
+		transform_output_box(&rect_clipped_region_box, data);
+		corner_location_transform(node_transform, &rect_clipped_corners);
+
 		struct fx_render_rect_options rect_options = {
 			.base = {
 				.box = dst_box,
@@ -1613,35 +1625,19 @@ static void scene_entry_render(struct render_list_entry *entry, const struct ren
 				},
 				.clip = &render_region,
 			},
+			.clipped_region = {
+				.area = rect_clipped_region_box,
+				.corner_radius = rect_clipped_region_corner_radius * data->scale,
+				.corners = rect_clipped_corners,
+			},
 		};
 
 		if (scene_rect->corner_radius && rect_corners != CORNER_LOCATION_NONE) {
-			struct wlr_box clipped_region_box = scene_rect->clipped_region.area;
-			int clipped_region_corner_radius = scene_rect->clipped_region.corner_radius;
-			enum corner_location clipped_corners = scene_rect->clipped_region.corners;
-
-			// Compensation
-			int node_x, node_y;
-			wlr_scene_node_coords(node, &node_x, &node_y);
-			clipped_region_box.x += node_x - node->x;
-			clipped_region_box.y += node_y - node->y;
-
-			clipped_region_box.x -= data->logical.x;
-			clipped_region_box.y -= data->logical.y;
-
-			scale_box(&clipped_region_box, data->scale);
-			transform_output_box(&clipped_region_box, data);
-			corner_location_transform(node_transform, &clipped_corners);
-
 			struct fx_render_rounded_rect_options rounded_rect_options = {
 				.base = rect_options.base,
 				.corner_radius = scene_rect->corner_radius * data->scale,
 				.corners = rect_corners,
-				.clipped_region = {
-					.area = clipped_region_box,
-					.corner_radius = clipped_region_corner_radius * data->scale,
-					.corners = clipped_corners,
-				},
+				.clipped_region = rect_options.clipped_region,
 			};
 			fx_render_pass_add_rounded_rect(data->render_pass, &rounded_rect_options);
 		} else {
@@ -1677,28 +1673,24 @@ static void scene_entry_render(struct render_list_entry *entry, const struct ren
 	case WLR_SCENE_NODE_SHADOW:;
 		struct wlr_scene_shadow *scene_shadow = wlr_scene_shadow_from_node(node);
 
-		struct wlr_box clipped_region_box = scene_shadow->clipped_region.area;
-		int clipped_region_corner_radius = scene_shadow->clipped_region.corner_radius;
-		enum corner_location clipped_corners = scene_shadow->clipped_region.corners;
+		struct wlr_box shadow_clipped_region_box = scene_shadow->clipped_region.area;
+		int shadow_clipped_region_corner_radius = scene_shadow->clipped_region.corner_radius;
+		enum corner_location shadow_clipped_corners = scene_shadow->clipped_region.corners;
 
-		// Compensation
-		int node_x, node_y;
-		wlr_scene_node_coords(node, &node_x, &node_y);
-		clipped_region_box.x += node_x - node->x;
-		clipped_region_box.y += node_y - node->y;
+		// Node relative -> Root relative
+		shadow_clipped_region_box.x += x;
+		shadow_clipped_region_box.y += y;
 
-		clipped_region_box.x -= data->logical.x;
-		clipped_region_box.y -= data->logical.y;
-		scale_box(&clipped_region_box, data->scale);
-		transform_output_box(&clipped_region_box, data);
-		corner_location_transform(node_transform, &clipped_corners);
+		scale_box(&shadow_clipped_region_box, data->scale);
+		transform_output_box(&shadow_clipped_region_box, data);
+		corner_location_transform(node_transform, &shadow_clipped_corners);
 
 		struct fx_render_box_shadow_options shadow_options = {
 			.box = dst_box,
 			.clipped_region = {
-				.area = clipped_region_box,
-				.corner_radius = clipped_region_corner_radius * data->scale,
-				.corners = clipped_corners,
+				.area = shadow_clipped_region_box,
+				.corner_radius = shadow_clipped_region_corner_radius * data->scale,
+				.corners = shadow_clipped_corners,
 			},
 			.blur_sigma = scene_shadow->blur_sigma,
 			.corner_radius = scene_shadow->corner_radius * data->scale,