Rework curseforge download (#611)

* Use the bulk endpoint on mod resolution for faster download
* Search on modrinth for api blocked mods
* Display a dialog for manually downloading blocked mods

4 files changed, 136 insertions, 41 deletions

diff --git a/launcher/modplatform/flame/FileResolvingTask.cpp b/launcher/modplatform/flame/FileResolvingTask.cpp
index 95924a68..a790ab9c 100644
--- a/launcher/modplatform/flame/FileResolvingTask.cpp
+++ b/launcher/modplatform/flame/FileResolvingTask.cpp
@@ -1,7 +1,9 @@
 #include "FileResolvingTask.h"
+
 #include "Json.h"
+#include "net/Upload.h"
 
-Flame::FileResolvingTask::FileResolvingTask(shared_qobject_ptr<QNetworkAccessManager> network, Flame::Manifest& toProcess)
+Flame::FileResolvingTask::FileResolvingTask(const shared_qobject_ptr<QNetworkAccessManager>& network, Flame::Manifest& toProcess)
     : m_network(network), m_toProcess(toProcess)
 {}
 
@@ -10,40 +12,116 @@ void Flame::FileResolvingTask::executeTask()
     setStatus(tr("Resolving mod IDs..."));
     setProgress(0, m_toProcess.files.size());
     m_dljob = new NetJob("Mod id resolver", m_network);
-    results.resize(m_toProcess.files.size());
-    int index = 0;
-    for (auto& file : m_toProcess.files) {
-        auto projectIdStr = QString::number(file.projectId);
-        auto fileIdStr = QString::number(file.fileId);
-        QString metaurl = QString("https://api.curseforge.com/v1/mods/%1/files/%2").arg(projectIdStr, fileIdStr);
-        auto dl = Net::Download::makeByteArray(QUrl(metaurl), &results[index]);
-        m_dljob->addNetAction(dl);
-        index++;
-    }
+    result.reset(new QByteArray());
+    //build json data to send
+    QJsonObject object;
+
+    object["fileIds"] = QJsonArray::fromVariantList(std::accumulate(m_toProcess.files.begin(), m_toProcess.files.end(), QVariantList(), [](QVariantList& l, const File& s) {
+        l.push_back(s.fileId);
+        return l;
+    }));
+    QByteArray data = Json::toText(object);
+    auto dl = Net::Upload::makeByteArray(QUrl("https://api.curseforge.com/v1/mods/files"), result.get(), data);
+    m_dljob->addNetAction(dl);
     connect(m_dljob.get(), &NetJob::finished, this, &Flame::FileResolvingTask::netJobFinished);
     m_dljob->start();
 }
 
 void Flame::FileResolvingTask::netJobFinished()
 {
-    bool failed = false;
     int index = 0;
-    for (auto& bytes : results) {
-        auto& out = m_toProcess.files[index];
+    // job to check modrinth for blocked projects
+    auto job = new NetJob("Modrinth check", m_network);
+    blockedProjects = QMap<File *,QByteArray *>();
+    auto doc = Json::requireDocument(*result);
+    auto array = Json::requireArray(doc.object()["data"]);
+    for (QJsonValueRef file : array) {
+        auto fileid = Json::requireInteger(Json::requireObject(file)["id"]);
+        auto& out = m_toProcess.files[fileid];
         try {
-            failed &= (!out.parseFromBytes(bytes));
+           out.parseFromObject(Json::requireObject(file));
         } catch (const JSONValidationError& e) {
-            qCritical() << "Resolving of" << out.projectId << out.fileId << "failed because of a parsing error:";
-            qCritical() << e.cause();
-            qCritical() << "JSON:";
-            qCritical() << bytes;
-            failed = true;
+            qDebug() << "Blocked mod on curseforge" << out.fileName;
+            auto hash = out.hash;
+            if(!hash.isEmpty()) {
+                auto url = QString("https://api.modrinth.com/v2/version_file/%1?algorithm=sha1").arg(hash);
+                auto output = new QByteArray();
+                auto dl = Net::Download::makeByteArray(QUrl(url), output);
+                QObject::connect(dl.get(), &Net::Download::succeeded, [&out]() {
+                    out.resolved = true;
+                });
+
+                job->addNetAction(dl);
+                blockedProjects.insert(&out, output);
+            }
         }
         index++;
     }
-    if (!failed) {
-        emitSucceeded();
+    connect(job, &NetJob::finished, this, &Flame::FileResolvingTask::modrinthCheckFinished);
+
+    job->start();
+}
+
+void Flame::FileResolvingTask::modrinthCheckFinished() {
+    qDebug() << "Finished with blocked mods : " << blockedProjects.size();
+
+    for (auto it = blockedProjects.keyBegin(); it != blockedProjects.keyEnd(); it++) {
+        auto &out = *it;
+        auto bytes = blockedProjects[out];
+        if (!out->resolved) {
+            delete bytes;
+            continue;
+        }
+        QJsonDocument doc = QJsonDocument::fromJson(*bytes);
+        auto obj = doc.object();
+        auto array = Json::requireArray(obj,"files");
+        for (auto file: array) {
+            auto fileObj = Json::requireObject(file);
+            auto primary = Json::requireBoolean(fileObj,"primary");
+            if (primary) {
+                out->url = Json::requireUrl(fileObj,"url");
+                qDebug() << "Found alternative on modrinth " << out->fileName;
+                break;
+            }
+        }
+        delete bytes;
+    }
+    //copy to an output list and filter out projects found on modrinth
+    auto block = new QList<File *>();
+    auto it = blockedProjects.keys();
+    std::copy_if(it.begin(), it.end(), std::back_inserter(*block), [](File *f) {
+        return !f->resolved;
+    });
+    //Display not found mods early
+    if (!block->empty()) {
+        //blocked mods found, we need the slug for displaying.... we need another job :D !
+        auto slugJob = new NetJob("Slug Job", m_network);
+        auto slugs = QVector<QByteArray>(block->size());
+        auto index = 0;
+        for (auto fileInfo: *block) {
+            auto projectId = fileInfo->projectId;
+            slugs[index] = QByteArray();
+            auto url = QString("https://api.curseforge.com/v1/mods/%1").arg(projectId);
+            auto dl = Net::Download::makeByteArray(url, &slugs[index]);
+            slugJob->addNetAction(dl);
+            index++;
+        }
+        connect(slugJob, &NetJob::succeeded, this, [slugs, this, slugJob, block]() {
+            slugJob->deleteLater();
+            auto index = 0;
+            for (const auto &slugResult: slugs) {
+                auto json = QJsonDocument::fromJson(slugResult);
+                auto base = Json::requireString(Json::requireObject(Json::requireObject(Json::requireObject(json),"data"),"links"),
+                        "websiteUrl");
+                auto mod = block->at(index);
+                auto link = QString("%1/download/%2").arg(base, QString::number(mod->fileId));
+                mod->websiteUrl = link;
+                index++;
+            }
+            emitSucceeded();
+        });
+        slugJob->start();
     } else {
-        emitFailed(tr("Some mod ID resolving tasks failed."));
+        emitSucceeded();
     }
 }
diff --git a/launcher/modplatform/flame/FileResolvingTask.h b/launcher/modplatform/flame/FileResolvingTask.h
index 5e5adcd7..87981f0a 100644
--- a/launcher/modplatform/flame/FileResolvingTask.h
+++ b/launcher/modplatform/flame/FileResolvingTask.h
@@ -10,7 +10,7 @@ class FileResolvingTask : public Task
 {
     Q_OBJECT
 public:
-    explicit FileResolvingTask(shared_qobject_ptr<QNetworkAccessManager> network, Flame::Manifest &toProcess);
+    explicit FileResolvingTask(const shared_qobject_ptr<QNetworkAccessManager>& network, Flame::Manifest &toProcess);
     virtual ~FileResolvingTask() {};
 
     const Flame::Manifest &getResults() const
@@ -27,7 +27,11 @@ protected slots:
 private: /* data */
     shared_qobject_ptr<QNetworkAccessManager> m_network;
     Flame::Manifest m_toProcess;
-    QVector<QByteArray> results;
+    std::shared_ptr<QByteArray> result;
     NetJob::Ptr m_dljob;
+
+    void modrinthCheckFinished();
+
+    QMap<File *, QByteArray *> blockedProjects;
 };
 }
diff --git a/launcher/modplatform/flame/PackManifest.cpp b/launcher/modplatform/flame/PackManifest.cpp
index e4f90c1a..12a4b990 100644
--- a/launcher/modplatform/flame/PackManifest.cpp
+++ b/launcher/modplatform/flame/PackManifest.cpp
@@ -41,7 +41,7 @@ static void loadManifestV1(Flame::Manifest& m, QJsonObject& manifest)
         auto obj = Json::requireObject(item);
         Flame::File file;
         loadFileV1(file, obj);
-        m.files.append(file);
+        m.files.insert(file.fileId,file);
     }
     m.overrides = Json::ensureString(manifest, "overrides", "overrides");
 }
@@ -61,21 +61,9 @@ void Flame::loadManifest(Flame::Manifest& m, const QString& filepath)
     loadManifestV1(m, obj);
 }
 
-bool Flame::File::parseFromBytes(const QByteArray& bytes)
+bool Flame::File::parseFromObject(const QJsonObject& obj)
 {
-    auto doc = Json::requireDocument(bytes);
-    if (!doc.isObject()) {
-        throw JSONValidationError(QString("data is not an object? that's not supposed to happen"));
-    }
-    auto obj = Json::ensureObject(doc.object(), "data");
-
     fileName = Json::requireString(obj, "fileName");
-
-    QString rawUrl = Json::requireString(obj, "downloadUrl");
-    url = QUrl(rawUrl, QUrl::TolerantMode);
-    if (!url.isValid()) {
-        throw JSONValidationError(QString("Invalid URL: %1").arg(rawUrl));
-    }
     // This is a piece of a Flame project JSON pulled out into the file metadata (here) for convenience
     // It is also optional
     type = File::Type::SingleFile;
@@ -87,6 +75,25 @@ bool Flame::File::parseFromBytes(const QByteArray& bytes)
         // this is probably a mod, dunno what else could modpacks download
         targetFolder = "mods";
     }
+    // get the hash
+    hash = QString();
+    auto hashes = Json::ensureArray(obj, "hashes");
+    for(QJsonValueRef item : hashes) {
+        auto hobj = Json::requireObject(item);
+        auto algo = Json::requireInteger(hobj, "algo");
+        auto value = Json::requireString(hobj, "value");
+        if (algo == 1) {
+            hash = value;
+        }
+    }
+
+
+    // may throw, if the project is blocked
+    QString rawUrl = Json::ensureString(obj, "downloadUrl");
+    url = QUrl(rawUrl, QUrl::TolerantMode);
+    if (!url.isValid()) {
+        throw JSONValidationError(QString("Invalid URL: %1").arg(rawUrl));
+    }
 
     resolved = true;
     return true;
diff --git a/launcher/modplatform/flame/PackManifest.h b/launcher/modplatform/flame/PackManifest.h
index 02f39f0e..26a48d1c 100644
--- a/launcher/modplatform/flame/PackManifest.h
+++ b/launcher/modplatform/flame/PackManifest.h
@@ -2,19 +2,24 @@
 
 #include <QString>
 #include <QVector>
+#include <QMap>
 #include <QUrl>
+#include <QJsonObject>
 
 namespace Flame
 {
 struct File
 {
     // NOTE: throws JSONValidationError
-    bool parseFromBytes(const QByteArray &bytes);
+    bool parseFromObject(const QJsonObject& object);
 
     int projectId = 0;
     int fileId = 0;
     // NOTE: the opposite to 'optional'. This is at the time of writing unused.
     bool required = true;
+    QString hash;
+    // NOTE: only set on blocked files ! Empty otherwise.
+    QString websiteUrl;
 
     // our
     bool resolved = false;
@@ -54,7 +59,8 @@ struct Manifest
     QString name;
     QString version;
     QString author;
-    QVector<Flame::File> files;
+    //File id -> File
+    QMap<int,Flame::File> files;
     QString overrides;
 };