From 14195835ef1dd25e10b0d873ec75c004b238ccba Mon Sep 17 00:00:00 2001
From: Vaxry <vaxry@vaxry.net>
Date: Thu, 9 Nov 2023 22:11:42 +0000
Subject: [PATCH] opengl: switch to black-and-white for alpha mattes

also fixes shadows on 10b
---
 src/render/OpenGL.cpp                         |  4 ++--
 src/render/OpenGL.hpp                         |  2 +-
 .../decorations/CHyprDropShadowDecoration.cpp | 19 +++++++++++++------
 src/render/shaders/Textures.hpp               |  2 +-
 4 files changed, 17 insertions(+), 10 deletions(-)

diff --git a/src/render/OpenGL.cpp b/src/render/OpenGL.cpp
index ab69295d0..26f3c1285 100644
--- a/src/render/OpenGL.cpp
+++ b/src/render/OpenGL.cpp
@@ -1718,7 +1718,7 @@ void CHyprOpenGLImpl::renderSnapshot(SLayerSurface** pLayer) {
     m_bEndFrame = false;
 }
 
-void CHyprOpenGLImpl::renderRoundedShadow(CBox* box, int round, int range, float a) {
+void CHyprOpenGLImpl::renderRoundedShadow(CBox* box, int round, int range, const CColor& color, float a) {
     RASSERT(m_RenderData.pMonitor, "Tried to render shadow without begin()!");
     RASSERT((box->width > 0 && box->height > 0), "Tried to render shadow with width/height < 0!");
     RASSERT(m_pCurrentWindow, "Tried to render shadow without a window!");
@@ -1737,7 +1737,7 @@ void CHyprOpenGLImpl::renderRoundedShadow(CBox* box, int round, int range, float
 
     const auto         SHADOWPOWER = std::clamp((int)*PSHADOWPOWER, 1, 4);
 
-    const auto         col = m_pCurrentWindow->m_cRealShadowColor.col();
+    const auto         col = color;
 
     float              matrix[9];
     wlr_matrix_project_box(matrix, box->pWlr(), wlr_output_transform_invert(!m_bEndFrame ? WL_OUTPUT_TRANSFORM_NORMAL : m_RenderData.pMonitor->transform), 0,
diff --git a/src/render/OpenGL.hpp b/src/render/OpenGL.hpp
index 0c6ef13cd..8c412302a 100644
--- a/src/render/OpenGL.hpp
+++ b/src/render/OpenGL.hpp
@@ -113,7 +113,7 @@ class CHyprOpenGLImpl {
     void               renderTexture(wlr_texture*, CBox*, float a, int round = 0, bool allowCustomUV = false);
     void               renderTexture(const CTexture&, CBox*, float a, int round = 0, bool discardActive = false, bool allowCustomUV = false);
     void               renderTextureWithBlur(const CTexture&, CBox*, float a, wlr_surface* pSurface, int round = 0, bool blockBlurOptimization = false, float blurA = 1.f);
-    void               renderRoundedShadow(CBox*, int round, int range, float a = 1.0);
+    void               renderRoundedShadow(CBox*, int round, int range, const CColor& color, float a = 1.0);
     void               renderBorder(CBox*, const CGradientValueData&, int round, int borderSize, float a = 1.0, int outerRound = -1 /* use round */);
     void               renderTextureMatte(const CTexture& tex, CBox* pBox, CFramebuffer& matte);
 
diff --git a/src/render/decorations/CHyprDropShadowDecoration.cpp b/src/render/decorations/CHyprDropShadowDecoration.cpp
index 5341dec9d..68a82c3a1 100644
--- a/src/render/decorations/CHyprDropShadowDecoration.cpp
+++ b/src/render/decorations/CHyprDropShadowDecoration.cpp
@@ -140,19 +140,26 @@ void CHyprDropShadowDecoration::draw(CMonitor* pMonitor, float a, const Vector2D
 
         windowBox.addExtents(SWindowDecorationExtents{m_eLastExtents * pMonitor->scale}.floor()).round();
 
-        if (windowBox.width < 1 || windowBox.height < 1) {
+        if (windowBox.width < 1 || windowBox.height < 1)
             return; // prevent assert failed
-        }
 
         alphaFB.bind();
-        g_pHyprOpenGL->clear(CColor(0, 0, 0, 0));
 
-        g_pHyprOpenGL->renderRect(&windowBox, CColor(1.0, 1.0, 1.0, 1.0), ROUNDING * pMonitor->scale);
+        // build the matte
+        // 10-bit formats have dogshit alpha channels, so we have to use the matte to its fullest.
+        // first, clear with black (fully transparent)
+        g_pHyprOpenGL->clear(CColor(0, 0, 0, 1));
+
+        // render white shadow
+        g_pHyprOpenGL->renderRoundedShadow(&fullBox, ROUNDING * pMonitor->scale, *PSHADOWSIZE * pMonitor->scale, CColor(1, 1, 1, 1), a);
+
+        // render black window box ("clip")
+        g_pHyprOpenGL->renderRect(&windowBox, CColor(0, 0, 0, 1.0), ROUNDING * pMonitor->scale);
 
         alphaSwapFB.bind();
-        g_pHyprOpenGL->clear(CColor(0, 0, 0, 0));
 
-        g_pHyprOpenGL->renderRoundedShadow(&fullBox, ROUNDING * pMonitor->scale, *PSHADOWSIZE * pMonitor->scale, a);
+        // alpha swap just has the shadow color. It will be the "texture" to render.
+        g_pHyprOpenGL->clear(m_pWindow->m_cRealShadowColor.col());
 
         LASTFB->bind();
 
diff --git a/src/render/shaders/Textures.hpp b/src/render/shaders/Textures.hpp
index 15e80410e..e0a2b4983 100644
--- a/src/render/shaders/Textures.hpp
+++ b/src/render/shaders/Textures.hpp
@@ -137,7 +137,7 @@ uniform sampler2D tex;
 uniform sampler2D texMatte;
 
 void main() {
-    gl_FragColor = texture2D(tex, v_texcoord) * (1.0 - texture2D(texMatte, v_texcoord)[3]);
+    gl_FragColor = texture2D(tex, v_texcoord) * texture2D(texMatte, v_texcoord)[0]; // I know it only uses R, but matte should be black/white anyways.
 })#";
 
 inline const std::string TEXFRAGSRCRGBX = R"#(