chore: replace naive mangohud detection with vulkan layer detection

Signed-off-by: Jan200101 <sentrycraft123@gmail.com>

4 files changed, 129 insertions, 11 deletions

diff --git a/launcher/Application.cpp b/launcher/Application.cpp
index 4ba9eb9b..8447083c 100644
--- a/launcher/Application.cpp
+++ b/launcher/Application.cpp
@@ -125,6 +125,7 @@
 #ifdef Q_OS_LINUX
 #include <dlfcn.h>
 #include "gamemode_client.h"
+#include "MangoHud.h"
 #endif
 
 
@@ -1519,17 +1520,8 @@ void Application::updateCapabilities()
     if (gamemode_query_status() >= 0)
         m_capabilities |= SupportsGameMode;
 
-    {
-        void *dummy = dlopen("libMangoHud_dlsym.so", RTLD_LAZY);
-        // try normal variant as well
-        if (dummy == NULL)
-            dummy = dlopen("libMangoHud.so", RTLD_LAZY);
-
-        if (dummy != NULL) {
-            dlclose(dummy);
-            m_capabilities |= SupportsMangoHud;
-        }
-    }
+    if (!MangoHud::getLibraryString().isEmpty())
+        m_capabilities |= SupportsMangoHud;
 #endif
 }
 
diff --git a/launcher/CMakeLists.txt b/launcher/CMakeLists.txt
index 3eb765dc..879c3157 100644
--- a/launcher/CMakeLists.txt
+++ b/launcher/CMakeLists.txt
@@ -90,6 +90,15 @@ set(CORE_SOURCES
     MMCTime.h
     MMCTime.cpp
 )
+if (UNIX AND NOT CYGWIN AND NOT APPLE)
+set(CORE_SOURCES
+    ${CORE_SOURCES}
+
+    # MangoHud
+    MangoHud.h
+    MangoHud.cpp
+    )
+endif()
 
 set(PATHMATCHER_SOURCES
     # Path matchers
diff --git a/launcher/MangoHud.cpp b/launcher/MangoHud.cpp
new file mode 100644
index 00000000..d635518e
--- /dev/null
+++ b/launcher/MangoHud.cpp
@@ -0,0 +1,90 @@
+// SPDX-License-Identifier: GPL-3.0-only
+/*
+ *  PrismLauncher - Minecraft Launcher
+ *  Copyright (C) 2022 Jan Drögehoff <sentrycraft123@gmail.com>
+ *
+ *  This program is free software: you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation, version 3.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+#include <QStringList>
+#include <QDir>
+#include <QString>
+#include <QtGlobal>
+
+#include "MangoHud.h"
+#include "FileSystem.h"
+#include "Json.h"
+
+namespace MangoHud {
+
+QString getLibraryString()
+{
+    /*
+     * Check for vulkan layers in this order:
+     *
+     * $VK_LAYER_PATH
+     * $XDG_DATA_DIRS (/usr/local/share/:/usr/share/)
+     * $XDG_DATA_HOME  (~/.local/share)
+     * /etc
+     * $XDG_CONFIG_DIRS (/etc/xdg)
+     * $XDG_CONFIG_HOME (~/.config)
+     */
+
+    QStringList vkLayerList;
+    {
+        QString home = QDir::homePath();
+
+        QString vkLayerPath = qEnvironmentVariable("VK_LAYER_PATH");
+        if (!vkLayerPath.isEmpty()) {
+            vkLayerList << vkLayerPath;
+        }
+
+        QStringList xdgDataDirs = qEnvironmentVariable("XDG_DATA_DIRS", "/usr/local/share/:/usr/share/").split(QLatin1String(":"));
+        for (QString dir : xdgDataDirs) {
+            vkLayerList << FS::PathCombine(dir, "vulkan", "implicit_layer.d");
+        }
+
+        QString xdgDataHome = qEnvironmentVariable("XDG_DATA_HOME");
+        if (xdgDataHome.isEmpty()) {
+            xdgDataHome = FS::PathCombine(home, ".local", "share");
+        }
+        vkLayerList << FS::PathCombine(xdgDataHome, "vulkan", "implicit_layer.d");
+
+        vkLayerList << "/etc";
+
+        QStringList xdgConfigDirs = qEnvironmentVariable("XDG_CONFIG_DIRS", "/etc/xdg").split(QLatin1String(":"));
+        for (QString dir : xdgConfigDirs) {
+            vkLayerList << FS::PathCombine(dir, "vulkan", "implicit_layer.d");
+        }
+
+        QString xdgConfigHome = qEnvironmentVariable("XDG_CONFIG_HOME");
+        if (xdgConfigHome.isEmpty()) {
+            xdgConfigHome = FS::PathCombine(home, ".config");
+        }
+        vkLayerList << FS::PathCombine(xdgConfigHome, "vulkan", "implicit_layer.d");
+    }
+
+    for (QString vkLayer : vkLayerList) {
+        QString filePath = FS::PathCombine(vkLayer, "MangoHud.json");
+        if (!QFile::exists(filePath))
+            continue;
+
+        auto conf = Json::requireDocument(filePath, vkLayer);
+        auto confObject = Json::requireObject(conf, vkLayer);
+        auto layer = Json::ensureObject(confObject, "layer");
+        return Json::ensureString(layer, "library_path");
+    }
+
+    return QString();
+}
+}  // namespace MangoHud
diff --git a/launcher/MangoHud.h b/launcher/MangoHud.h
new file mode 100644
index 00000000..7b7c2849
--- /dev/null
+++ b/launcher/MangoHud.h
@@ -0,0 +1,27 @@
+// SPDX-License-Identifier: GPL-3.0-only
+/*
+ *  PrismLauncher - Minecraft Launcher
+ *  Copyright (C) 2022 Jan Drögehoff <sentrycraft123@gmail.com>
+ *
+ *  This program is free software: you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation, version 3.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+#pragma once
+
+#include <QString>
+#include <QStringList>
+
+namespace MangoHud {
+
+QString getLibraryString();
+}