From 844b2457769d61131f97b5e82bb134568dfd42ed Mon Sep 17 00:00:00 2001 From: flow Date: Fri, 3 Jun 2022 19:08:01 -0300 Subject: feat: add EnsureMetadataTask This task is responsible for checking if the mod has metadata for a specific provider, and create it if it doesn't. In the context of the mod updater, this is not the best architecture, since we do a single task for each mod. However, this way of structuring it allows us to use it later on in more diverse scenarios. This way we decouple this task from the mod updater, trading off some performance (though that will be mitigated when we have a way of running arbitrary tasks concurrently). Signed-off-by: flow --- launcher/modplatform/EnsureMetadataTask.cpp | 244 ++++++++++++++++++++++++++++ 1 file changed, 244 insertions(+) create mode 100644 launcher/modplatform/EnsureMetadataTask.cpp (limited to 'launcher/modplatform/EnsureMetadataTask.cpp') diff --git a/launcher/modplatform/EnsureMetadataTask.cpp b/launcher/modplatform/EnsureMetadataTask.cpp new file mode 100644 index 00000000..dc92d8ab --- /dev/null +++ b/launcher/modplatform/EnsureMetadataTask.cpp @@ -0,0 +1,244 @@ +#include "EnsureMetadataTask.h" + +#include +#include + +#include "FileSystem.h" +#include "Json.h" +#include "minecraft/mod/Mod.h" +#include "minecraft/mod/tasks/LocalModUpdateTask.h" +#include "modplatform/flame/FlameAPI.h" +#include "modplatform/flame/FlameModIndex.h" +#include "modplatform/modrinth/ModrinthAPI.h" +#include "modplatform/modrinth/ModrinthPackIndex.h" +#include "net/NetJob.h" +#include "tasks/MultipleOptionsTask.h" + +static ModPlatform::ProviderCapabilities ProviderCaps; + +static ModrinthAPI modrinth_api; +static FlameAPI flame_api; + +EnsureMetadataTask::EnsureMetadataTask(Mod& mod, QDir& dir, bool try_all, ModPlatform::Provider prov) + : m_mod(mod), m_index_dir(dir), m_provider(prov), m_try_all(try_all) +{} + +bool EnsureMetadataTask::abort() +{ + return m_task_handler->abort(); +} + +void EnsureMetadataTask::executeTask() +{ + // They already have the right metadata :o + if (m_mod.status() != ModStatus::NoMetadata && m_mod.metadata() && m_mod.metadata()->provider == m_provider) { + emitReady(); + return; + } + + // Folders don't have metadata + if (m_mod.type() == Mod::MOD_FOLDER) { + emitReady(); + return; + } + + setStatus(tr("Generating %1's metadata...").arg(m_mod.name())); + qDebug() << QString("Generating %1's metadata...").arg(m_mod.name()); + + QByteArray jar_data; + + try { + jar_data = FS::read(m_mod.fileinfo().absoluteFilePath()); + } catch (FS::FileSystemException& e) { + qCritical() << QString("Failed to open / read JAR file of %1").arg(m_mod.name()); + qCritical() << QString("Reason: ") << e.cause(); + + emitFail(); + return; + } + + auto tsk = new MultipleOptionsTask(nullptr, "GetMetadataTask"); + + switch (m_provider) { + case (ModPlatform::Provider::MODRINTH): + modrinthEnsureMetadata(*tsk, jar_data); + if (m_try_all) + flameEnsureMetadata(*tsk, jar_data); + + break; + case (ModPlatform::Provider::FLAME): + flameEnsureMetadata(*tsk, jar_data); + if (m_try_all) + modrinthEnsureMetadata(*tsk, jar_data); + + break; + } + + connect(tsk, &MultipleOptionsTask::finished, this, [tsk] { tsk->deleteLater(); }); + connect(tsk, &MultipleOptionsTask::failed, [this] { + qCritical() << QString("Download of %1's metadata failed").arg(m_mod.name()); + + emitFail(); + }); + connect(tsk, &MultipleOptionsTask::succeeded, this, &EnsureMetadataTask::emitReady); + + m_task_handler = tsk; + + tsk->start(); +} + +void EnsureMetadataTask::emitReady() +{ + emit metadataReady(); + emitSucceeded(); +} + +void EnsureMetadataTask::emitFail() +{ + qDebug() << QString("Failed to generate metadata for %1").arg(m_mod.name()); + emit metadataFailed(); + //emitFailed(tr("Failed to generate metadata for %1").arg(m_mod.name())); + emitSucceeded(); +} + +void EnsureMetadataTask::modrinthEnsureMetadata(SequentialTask& tsk, QByteArray& jar_data) +{ + // Modrinth currently garantees that some hash types will always be present. + // But let's be sure and cover all cases anyways :) + for (auto hash_type : ProviderCaps.hashType(ModPlatform::Provider::MODRINTH)) { + auto* response = new QByteArray(); + auto hash = QString(ProviderCaps.hash(ModPlatform::Provider::MODRINTH, jar_data, hash_type).toHex()); + auto ver_task = modrinth_api.currentVersion(hash, hash_type, response); + + // Prevents unfortunate timings when aborting the task + if (!ver_task) + return; + + connect(ver_task.get(), &NetJob::succeeded, this, [this, ver_task, response] { + QJsonParseError parse_error{}; + QJsonDocument doc = QJsonDocument::fromJson(*response, &parse_error); + if (parse_error.error != QJsonParseError::NoError) { + qWarning() << "Error while parsing JSON response from " << m_mod.name() << " at " << parse_error.offset + << " reason: " << parse_error.errorString(); + qWarning() << *response; + + ver_task->failed(parse_error.errorString()); + return; + } + + auto doc_obj = Json::requireObject(doc); + auto ver = Modrinth::loadIndexedPackVersion(doc_obj, {}, m_mod.fileinfo().fileName()); + + // Minimal IndexedPack to create the metadata + ModPlatform::IndexedPack pack; + pack.name = m_mod.name(); + pack.provider = ModPlatform::Provider::MODRINTH; + pack.addonId = ver.addonId; + + // Prevent file name mismatch + ver.fileName = m_mod.fileinfo().fileName(); + + QDir tmp_index_dir(m_index_dir); + + { + LocalModUpdateTask update_metadata(m_index_dir, pack, ver); + QEventLoop loop; + QTimer timeout; + + QObject::connect(&update_metadata, &Task::finished, &loop, &QEventLoop::quit); + QObject::connect(&timeout, &QTimer::timeout, &loop, &QEventLoop::quit); + + update_metadata.start(); + timeout.start(100); + + loop.exec(); + } + + auto mod_name = m_mod.name(); + auto meta = new Metadata::ModStruct(Metadata::get(tmp_index_dir, mod_name)); + m_mod.setMetadata(meta); + }); + + tsk.addTask(ver_task); + } +} + +void EnsureMetadataTask::flameEnsureMetadata(SequentialTask& tsk, QByteArray& jar_data) +{ + QByteArray jar_data_treated; + for (char c : jar_data) { + // CF-specific + if (!(c == 9 || c == 10 || c == 13 || c == 32)) + jar_data_treated.push_back(c); + } + + auto* response = new QByteArray(); + + std::list fingerprints; + auto murmur = MurmurHash2(jar_data_treated, jar_data_treated.length()); + fingerprints.push_back(murmur); + + auto ver_task = flame_api.matchFingerprints(fingerprints, response); + + connect(ver_task.get(), &Task::succeeded, this, [this, ver_task, response] { + QDir tmp_index_dir(m_index_dir); + + QJsonParseError parse_error{}; + QJsonDocument doc = QJsonDocument::fromJson(*response, &parse_error); + if (parse_error.error != QJsonParseError::NoError) { + qWarning() << "Error while parsing JSON response from " << m_mod.name() << " at " << parse_error.offset + << " reason: " << parse_error.errorString(); + qWarning() << *response; + + ver_task->failed(parse_error.errorString()); + return; + } + + try { + auto doc_obj = Json::requireObject(doc); + auto data_obj = Json::ensureObject(doc_obj, "data"); + auto match_obj = Json::ensureObject(Json::ensureArray(data_obj, "exactMatches")[0], {}); + if (match_obj.isEmpty()) { + qCritical() << "Fingerprint match is empty!"; + + ver_task->failed(parse_error.errorString()); + return; + } + + auto file_obj = Json::ensureObject(match_obj, "file"); + + ModPlatform::IndexedPack pack; + pack.name = m_mod.name(); + pack.provider = ModPlatform::Provider::FLAME; + pack.addonId = Json::requireInteger(file_obj, "modId"); + + ModPlatform::IndexedVersion ver = FlameMod::loadIndexedPackVersion(file_obj); + + // Prevent file name mismatch + ver.fileName = m_mod.fileinfo().fileName(); + + { + LocalModUpdateTask update_metadata(m_index_dir, pack, ver); + QEventLoop loop; + QTimer timeout; + + QObject::connect(&update_metadata, &Task::finished, &loop, &QEventLoop::quit); + QObject::connect(&timeout, &QTimer::timeout, &loop, &QEventLoop::quit); + + update_metadata.start(); + timeout.start(100); + + loop.exec(); + } + + auto mod_name = m_mod.name(); + auto meta = new Metadata::ModStruct(Metadata::get(tmp_index_dir, mod_name)); + m_mod.setMetadata(meta); + + } catch (Json::JsonException& e) { + emitFailed(e.cause() + " : " + e.what()); + } + }); + + tsk.addTask(ver_task); +} -- cgit From 4e6978ff6f61777a2e2e989cba58a9f0c48d2782 Mon Sep 17 00:00:00 2001 From: flow Date: Thu, 16 Jun 2022 11:45:29 -0300 Subject: feat: improve metadata gen. networking and performance This makes the metadata generation code a lot messier and harder to use, but there's not really much else that can be done about it while preserving all it's capabilities :( At least we now have speed Signed-off-by: flow --- launcher/modplatform/EnsureMetadataTask.cpp | 550 +++++++++++++++++++++------- 1 file changed, 413 insertions(+), 137 deletions(-) (limited to 'launcher/modplatform/EnsureMetadataTask.cpp') diff --git a/launcher/modplatform/EnsureMetadataTask.cpp b/launcher/modplatform/EnsureMetadataTask.cpp index dc92d8ab..cf4e55b9 100644 --- a/launcher/modplatform/EnsureMetadataTask.cpp +++ b/launcher/modplatform/EnsureMetadataTask.cpp @@ -19,226 +19,502 @@ static ModPlatform::ProviderCapabilities ProviderCaps; static ModrinthAPI modrinth_api; static FlameAPI flame_api; -EnsureMetadataTask::EnsureMetadataTask(Mod& mod, QDir& dir, bool try_all, ModPlatform::Provider prov) - : m_mod(mod), m_index_dir(dir), m_provider(prov), m_try_all(try_all) -{} - -bool EnsureMetadataTask::abort() +EnsureMetadataTask::EnsureMetadataTask(Mod& mod, QDir dir, ModPlatform::Provider prov) : Task(nullptr), m_index_dir(dir), m_provider(prov) { - return m_task_handler->abort(); + auto hash = getHash(mod); + if (hash.isEmpty()) + emitFail(mod); + else + m_mods.insert(hash, mod); } -void EnsureMetadataTask::executeTask() +EnsureMetadataTask::EnsureMetadataTask(std::list& mods, QDir dir, ModPlatform::Provider prov) + : Task(nullptr), m_index_dir(dir), m_provider(prov) { - // They already have the right metadata :o - if (m_mod.status() != ModStatus::NoMetadata && m_mod.metadata() && m_mod.metadata()->provider == m_provider) { - emitReady(); - return; - } + for (auto& mod : mods) { + if (!mod.valid()) { + emitFail(mod); + continue; + } - // Folders don't have metadata - if (m_mod.type() == Mod::MOD_FOLDER) { - emitReady(); - return; - } + auto hash = getHash(mod); + if (hash.isEmpty()) { + emitFail(mod); + continue; + } - setStatus(tr("Generating %1's metadata...").arg(m_mod.name())); - qDebug() << QString("Generating %1's metadata...").arg(m_mod.name()); + m_mods.insert(hash, mod); + } +} +QString EnsureMetadataTask::getHash(Mod& mod) +{ + /* Here we create a mapping hash -> mod, because we need that relationship to parse the API routes */ QByteArray jar_data; - try { - jar_data = FS::read(m_mod.fileinfo().absoluteFilePath()); + jar_data = FS::read(mod.fileinfo().absoluteFilePath()); } catch (FS::FileSystemException& e) { - qCritical() << QString("Failed to open / read JAR file of %1").arg(m_mod.name()); + qCritical() << QString("Failed to open / read JAR file of %1").arg(mod.name()); qCritical() << QString("Reason: ") << e.cause(); - emitFail(); - return; + return {}; } - auto tsk = new MultipleOptionsTask(nullptr, "GetMetadataTask"); + switch (m_provider) { + case ModPlatform::Provider::MODRINTH: { + auto hash_type = ProviderCaps.hashType(ModPlatform::Provider::MODRINTH).first(); + + return QString(ProviderCaps.hash(ModPlatform::Provider::MODRINTH, jar_data, hash_type).toHex()); + } + case ModPlatform::Provider::FLAME: { + QByteArray jar_data_treated; + for (char c : jar_data) { + // CF-specific + if (!(c == 9 || c == 10 || c == 13 || c == 32)) + jar_data_treated.push_back(c); + } + + return QString::number(MurmurHash2(jar_data_treated, jar_data_treated.length())); + } + } + + return {}; +} + +bool EnsureMetadataTask::abort() +{ + // Prevent sending signals to a dead object + disconnect(this, 0, 0, 0); + + if (m_current_task) + return m_current_task->abort(); + return true; +} + +void EnsureMetadataTask::executeTask() +{ + setStatus(tr("Checking if mods have metadata...")); + + for (auto mod : m_mods) { + if (!mod.valid()) + continue; + + // They already have the right metadata :o + if (mod.status() != ModStatus::NoMetadata && mod.metadata() && mod.metadata()->provider == m_provider) { + qDebug() << "Mod" << mod.name() << "already has metadata!"; + emitReady(mod); + return; + } + + // Folders don't have metadata + if (mod.type() == Mod::MOD_FOLDER) { + emitReady(mod); + return; + } + } + + NetJob::Ptr version_task; switch (m_provider) { case (ModPlatform::Provider::MODRINTH): - modrinthEnsureMetadata(*tsk, jar_data); - if (m_try_all) - flameEnsureMetadata(*tsk, jar_data); - + version_task = modrinthVersionsTask(); break; case (ModPlatform::Provider::FLAME): - flameEnsureMetadata(*tsk, jar_data); - if (m_try_all) - modrinthEnsureMetadata(*tsk, jar_data); - + version_task = flameVersionsTask(); break; } - connect(tsk, &MultipleOptionsTask::finished, this, [tsk] { tsk->deleteLater(); }); - connect(tsk, &MultipleOptionsTask::failed, [this] { - qCritical() << QString("Download of %1's metadata failed").arg(m_mod.name()); + auto invalidade_leftover = [this] { + QMutableHashIterator mods_iter(m_mods); + while (mods_iter.hasNext()) { + auto mod = mods_iter.next(); + emitFail(mod.value()); + } + + emitSucceeded(); + }; + + connect(version_task.get(), &Task::finished, this, [this, invalidade_leftover] { + NetJob::Ptr project_task; + + switch (m_provider) { + case (ModPlatform::Provider::MODRINTH): + project_task = modrinthProjectsTask(); + break; + case (ModPlatform::Provider::FLAME): + project_task = flameProjectsTask(); + break; + } + + if (!project_task) { + invalidade_leftover(); + return; + } + + connect(project_task.get(), &Task::finished, this, [=] { + invalidade_leftover(); + project_task->deleteLater(); + m_current_task = nullptr; + }); - emitFail(); + m_current_task = project_task.get(); + project_task->start(); }); - connect(tsk, &MultipleOptionsTask::succeeded, this, &EnsureMetadataTask::emitReady); - m_task_handler = tsk; + connect(version_task.get(), &Task::finished, [=] { + version_task->deleteLater(); + m_current_task = nullptr; + }); + + if (m_mods.size() > 1) + setStatus(tr("Requesting metadata information from %1...").arg(ProviderCaps.readableName(m_provider))); + else if (!m_mods.empty()) + setStatus(tr("Requesting metadata information from %1 for '%2'...") + .arg(ProviderCaps.readableName(m_provider), m_mods.begin().value().name())); - tsk->start(); + m_current_task = version_task.get(); + version_task->start(); } -void EnsureMetadataTask::emitReady() +void EnsureMetadataTask::emitReady(Mod& m) { - emit metadataReady(); - emitSucceeded(); + qDebug() << QString("Generated metadata for %1").arg(m.name()); + emit metadataReady(m); + + m_mods.remove(getHash(m)); } -void EnsureMetadataTask::emitFail() +void EnsureMetadataTask::emitFail(Mod& m) { - qDebug() << QString("Failed to generate metadata for %1").arg(m_mod.name()); - emit metadataFailed(); - //emitFailed(tr("Failed to generate metadata for %1").arg(m_mod.name())); - emitSucceeded(); + qDebug() << QString("Failed to generate metadata for %1").arg(m.name()); + emit metadataFailed(m); + + m_mods.remove(getHash(m)); } -void EnsureMetadataTask::modrinthEnsureMetadata(SequentialTask& tsk, QByteArray& jar_data) +// Modrinth + +NetJob::Ptr EnsureMetadataTask::modrinthVersionsTask() { - // Modrinth currently garantees that some hash types will always be present. - // But let's be sure and cover all cases anyways :) - for (auto hash_type : ProviderCaps.hashType(ModPlatform::Provider::MODRINTH)) { - auto* response = new QByteArray(); - auto hash = QString(ProviderCaps.hash(ModPlatform::Provider::MODRINTH, jar_data, hash_type).toHex()); - auto ver_task = modrinth_api.currentVersion(hash, hash_type, response); - - // Prevents unfortunate timings when aborting the task - if (!ver_task) - return; + auto hash_type = ProviderCaps.hashType(ModPlatform::Provider::MODRINTH).first(); - connect(ver_task.get(), &NetJob::succeeded, this, [this, ver_task, response] { - QJsonParseError parse_error{}; - QJsonDocument doc = QJsonDocument::fromJson(*response, &parse_error); - if (parse_error.error != QJsonParseError::NoError) { - qWarning() << "Error while parsing JSON response from " << m_mod.name() << " at " << parse_error.offset - << " reason: " << parse_error.errorString(); - qWarning() << *response; + auto* response = new QByteArray(); + auto ver_task = modrinth_api.currentVersions(m_mods.keys(), hash_type, response); - ver_task->failed(parse_error.errorString()); - return; + // Prevents unfortunate timings when aborting the task + if (!ver_task) + return {}; + + connect(ver_task.get(), &NetJob::succeeded, this, [this, response] { + QJsonParseError parse_error{}; + QJsonDocument doc = QJsonDocument::fromJson(*response, &parse_error); + if (parse_error.error != QJsonParseError::NoError) { + qWarning() << "Error while parsing JSON response from Modrinth::CurrentVersions at " << parse_error.offset + << " reason: " << parse_error.errorString(); + qWarning() << *response; + + failed(parse_error.errorString()); + return; + } + + try { + auto entries = Json::requireObject(doc); + for (auto& hash : m_mods.keys()) { + auto mod = m_mods.find(hash).value(); + try { + auto entry = Json::requireObject(entries, hash); + + setStatus(tr("Parsing API response from Modrinth for '%1'...").arg(mod.name())); + qDebug() << "Getting version for" << mod.name() << "from Modrinth"; + + m_temp_versions.insert(hash, Modrinth::loadIndexedPackVersion(entry)); + } catch (Json::JsonException& e) { + qDebug() << e.cause(); + qDebug() << entries; + + emitFail(mod); + } } + } catch (Json::JsonException& e) { + qDebug() << e.cause(); + qDebug() << doc; + } + }); - auto doc_obj = Json::requireObject(doc); - auto ver = Modrinth::loadIndexedPackVersion(doc_obj, {}, m_mod.fileinfo().fileName()); + return ver_task; +} + +NetJob::Ptr EnsureMetadataTask::modrinthProjectsTask() +{ + QHash addonIds; + for (auto const& data : m_temp_versions) + addonIds.insert(data.addonId.toString(), data.hash); + + auto response = new QByteArray(); + NetJob::Ptr proj_task; + + if (addonIds.isEmpty()) { + qWarning() << "No addonId found!"; + } else if (addonIds.size() == 1) { + proj_task = modrinth_api.getProject(*addonIds.keyBegin(), response); + } else { + proj_task = modrinth_api.getProjects(addonIds.keys(), response); + } + + // Prevents unfortunate timings when aborting the task + if (!proj_task) + return {}; + + connect(proj_task.get(), &NetJob::succeeded, this, [this, response, addonIds] { + QJsonParseError parse_error{}; + auto doc = QJsonDocument::fromJson(*response, &parse_error); + if (parse_error.error != QJsonParseError::NoError) { + qWarning() << "Error while parsing JSON response from Modrinth projects task at " << parse_error.offset + << " reason: " << parse_error.errorString(); + qWarning() << *response; + return; + } - // Minimal IndexedPack to create the metadata - ModPlatform::IndexedPack pack; - pack.name = m_mod.name(); - pack.provider = ModPlatform::Provider::MODRINTH; - pack.addonId = ver.addonId; + try { + QJsonArray entries; + if (addonIds.size() == 1) + entries = { doc.object() }; + else + entries = Json::requireArray(doc); - // Prevent file name mismatch - ver.fileName = m_mod.fileinfo().fileName(); + for (auto entry : entries) { + auto entry_obj = Json::requireObject(entry); + auto entry_id = Json::requireString(entry_obj, "id"); - QDir tmp_index_dir(m_index_dir); + auto hash = addonIds.find(entry_id).value(); - { - LocalModUpdateTask update_metadata(m_index_dir, pack, ver); - QEventLoop loop; - QTimer timeout; + auto mod = m_mods.find(hash).value(); - QObject::connect(&update_metadata, &Task::finished, &loop, &QEventLoop::quit); - QObject::connect(&timeout, &QTimer::timeout, &loop, &QEventLoop::quit); + try { + setStatus(tr("Parsing API response from Modrinth for '%1'...").arg(mod.name())); - update_metadata.start(); - timeout.start(100); + ModPlatform::IndexedPack pack; + Modrinth::loadIndexedPack(pack, entry_obj); - loop.exec(); - } + modrinthCallback(pack, m_temp_versions.find(hash).value(), mod); + } catch (Json::JsonException& e) { + qDebug() << e.cause(); + qDebug() << entries; - auto mod_name = m_mod.name(); - auto meta = new Metadata::ModStruct(Metadata::get(tmp_index_dir, mod_name)); - m_mod.setMetadata(meta); - }); + emitFail(mod); + } + } + } catch (Json::JsonException& e) { + qDebug() << e.cause(); + qDebug() << doc; + } + }); - tsk.addTask(ver_task); - } + return proj_task; } -void EnsureMetadataTask::flameEnsureMetadata(SequentialTask& tsk, QByteArray& jar_data) +// Flame +NetJob::Ptr EnsureMetadataTask::flameVersionsTask() { - QByteArray jar_data_treated; - for (char c : jar_data) { - // CF-specific - if (!(c == 9 || c == 10 || c == 13 || c == 32)) - jar_data_treated.push_back(c); - } - auto* response = new QByteArray(); std::list fingerprints; - auto murmur = MurmurHash2(jar_data_treated, jar_data_treated.length()); - fingerprints.push_back(murmur); + for (auto& murmur : m_mods.keys()) { + fingerprints.push_back(murmur.toUInt()); + } auto ver_task = flame_api.matchFingerprints(fingerprints, response); - connect(ver_task.get(), &Task::succeeded, this, [this, ver_task, response] { - QDir tmp_index_dir(m_index_dir); - + connect(ver_task.get(), &Task::succeeded, this, [this, response] { QJsonParseError parse_error{}; QJsonDocument doc = QJsonDocument::fromJson(*response, &parse_error); if (parse_error.error != QJsonParseError::NoError) { - qWarning() << "Error while parsing JSON response from " << m_mod.name() << " at " << parse_error.offset + qWarning() << "Error while parsing JSON response from Modrinth::CurrentVersions at " << parse_error.offset << " reason: " << parse_error.errorString(); qWarning() << *response; - ver_task->failed(parse_error.errorString()); + failed(parse_error.errorString()); return; } try { auto doc_obj = Json::requireObject(doc); - auto data_obj = Json::ensureObject(doc_obj, "data"); - auto match_obj = Json::ensureObject(Json::ensureArray(data_obj, "exactMatches")[0], {}); - if (match_obj.isEmpty()) { - qCritical() << "Fingerprint match is empty!"; + auto data_obj = Json::requireObject(doc_obj, "data"); + auto data_arr = Json::requireArray(data_obj, "exactMatches"); + + if (data_arr.isEmpty()) { + qWarning() << "No matches found for fingerprint search!"; - ver_task->failed(parse_error.errorString()); return; } - auto file_obj = Json::ensureObject(match_obj, "file"); + for (auto match : data_arr) { + auto match_obj = Json::ensureObject(match, {}); + auto file_obj = Json::ensureObject(match_obj, "file", {}); - ModPlatform::IndexedPack pack; - pack.name = m_mod.name(); - pack.provider = ModPlatform::Provider::FLAME; - pack.addonId = Json::requireInteger(file_obj, "modId"); + if (match_obj.isEmpty() || file_obj.isEmpty()) { + qWarning() << "Fingerprint match is empty!"; - ModPlatform::IndexedVersion ver = FlameMod::loadIndexedPackVersion(file_obj); + return; + } - // Prevent file name mismatch - ver.fileName = m_mod.fileinfo().fileName(); + auto fingerprint = QString::number(Json::ensureVariant(file_obj, "fileFingerprint").toUInt()); + auto mod = m_mods.find(fingerprint); + if (mod == m_mods.end()) { + qWarning() << "Invalid fingerprint from the API response."; + continue; + } - { - LocalModUpdateTask update_metadata(m_index_dir, pack, ver); - QEventLoop loop; - QTimer timeout; + setStatus(tr("Parsing API response from CurseForge for '%1'...").arg(mod->name())); - QObject::connect(&update_metadata, &Task::finished, &loop, &QEventLoop::quit); - QObject::connect(&timeout, &QTimer::timeout, &loop, &QEventLoop::quit); + m_temp_versions.insert(fingerprint, FlameMod::loadIndexedPackVersion(file_obj)); + } - update_metadata.start(); - timeout.start(100); + } catch (Json::JsonException& e) { + qDebug() << e.cause(); + qDebug() << doc; + } + }); - loop.exec(); - } + return ver_task; +} + +NetJob::Ptr EnsureMetadataTask::flameProjectsTask() +{ + QHash addonIds; + for (auto const& hash : m_mods.keys()) { + if (m_temp_versions.contains(hash)) { + auto const& data = m_temp_versions.find(hash).value(); + addonIds.insert(data.addonId.toString(), hash); + } + } + + auto response = new QByteArray(); + NetJob::Ptr proj_task; + + if (addonIds.isEmpty()) { + qWarning() << "No addonId found!"; + } else if (addonIds.size() == 1) { + proj_task = flame_api.getProject(*addonIds.keyBegin(), response); + } else { + proj_task = flame_api.getProjects(addonIds.keys(), response); + } + + // Prevents unfortunate timings when aborting the task + if (!proj_task) + return {}; + + connect(proj_task.get(), &NetJob::succeeded, this, [this, response, addonIds] { + QJsonParseError parse_error{}; + auto doc = QJsonDocument::fromJson(*response, &parse_error); + if (parse_error.error != QJsonParseError::NoError) { + qWarning() << "Error while parsing JSON response from Modrinth projects task at " << parse_error.offset + << " reason: " << parse_error.errorString(); + qWarning() << *response; + return; + } + + try { + QJsonArray entries; + if (addonIds.size() == 1) + entries = { Json::requireObject(Json::requireObject(doc), "data") }; + else + entries = Json::requireArray(Json::requireObject(doc), "data"); + + for (auto entry : entries) { + auto entry_obj = Json::requireObject(entry); + + auto id = QString::number(Json::requireInteger(entry_obj, "id")); + auto hash = addonIds.find(id).value(); + auto mod = m_mods.find(hash).value(); + + try { + setStatus(tr("Parsing API response from CurseForge for '%1'...").arg(mod.name())); - auto mod_name = m_mod.name(); - auto meta = new Metadata::ModStruct(Metadata::get(tmp_index_dir, mod_name)); - m_mod.setMetadata(meta); + ModPlatform::IndexedPack pack; + FlameMod::loadIndexedPack(pack, entry_obj); + flameCallback(pack, m_temp_versions.find(hash).value(), mod); + } catch (Json::JsonException& e) { + qDebug() << e.cause(); + qDebug() << entries; + + emitFail(mod); + } + } } catch (Json::JsonException& e) { - emitFailed(e.cause() + " : " + e.what()); + qDebug() << e.cause(); + qDebug() << doc; } }); - tsk.addTask(ver_task); + return proj_task; +} + +void EnsureMetadataTask::modrinthCallback(ModPlatform::IndexedPack& pack, ModPlatform::IndexedVersion& ver, Mod& mod) +{ + // Prevent file name mismatch + ver.fileName = mod.fileinfo().fileName(); + + QDir tmp_index_dir(m_index_dir); + + { + LocalModUpdateTask update_metadata(m_index_dir, pack, ver); + QEventLoop loop; + + QObject::connect(&update_metadata, &Task::finished, &loop, &QEventLoop::quit); + + update_metadata.start(); + + if (!update_metadata.isFinished()) + loop.exec(); + } + + auto metadata = Metadata::get(tmp_index_dir, pack.slug); + if (!metadata.isValid()) { + qCritical() << "Failed to generate metadata at last step!"; + emitFail(mod); + return; + } + + mod.setMetadata(metadata); + + emitReady(mod); +} + +void EnsureMetadataTask::flameCallback(ModPlatform::IndexedPack& pack, ModPlatform::IndexedVersion& ver, Mod& mod) +{ + try { + // Prevent file name mismatch + ver.fileName = mod.fileinfo().fileName(); + + QDir tmp_index_dir(m_index_dir); + + { + LocalModUpdateTask update_metadata(m_index_dir, pack, ver); + QEventLoop loop; + + QObject::connect(&update_metadata, &Task::finished, &loop, &QEventLoop::quit); + + update_metadata.start(); + + if (!update_metadata.isFinished()) + loop.exec(); + } + + auto metadata = Metadata::get(tmp_index_dir, pack.slug); + if (!metadata.isValid()) { + qCritical() << "Failed to generate metadata at last step!"; + emitFail(mod); + return; + } + + mod.setMetadata(metadata); + + emitReady(mod); + } catch (Json::JsonException& e) { + qDebug() << e.cause(); + + emitFail(mod); + } } -- cgit From 5f75e531e61e1f2cb5d602e084e9a0ddd1c85a5c Mon Sep 17 00:00:00 2001 From: flow Date: Mon, 20 Jun 2022 08:55:35 -0300 Subject: fix: handling around disabled mods Don't update disabled mods to prevent mod duplication. Also, chop filename in the metadata with a '.disabled'. Signed-off-by: flow --- launcher/modplatform/EnsureMetadataTask.cpp | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'launcher/modplatform/EnsureMetadataTask.cpp') diff --git a/launcher/modplatform/EnsureMetadataTask.cpp b/launcher/modplatform/EnsureMetadataTask.cpp index cf4e55b9..19e44ce0 100644 --- a/launcher/modplatform/EnsureMetadataTask.cpp +++ b/launcher/modplatform/EnsureMetadataTask.cpp @@ -455,6 +455,8 @@ void EnsureMetadataTask::modrinthCallback(ModPlatform::IndexedPack& pack, ModPla { // Prevent file name mismatch ver.fileName = mod.fileinfo().fileName(); + if (ver.fileName.endsWith(".disabled")) + ver.fileName.chop(9); QDir tmp_index_dir(m_index_dir); @@ -487,6 +489,8 @@ void EnsureMetadataTask::flameCallback(ModPlatform::IndexedPack& pack, ModPlatfo try { // Prevent file name mismatch ver.fileName = mod.fileinfo().fileName(); + if (ver.fileName.endsWith(".disabled")) + ver.fileName.chop(9); QDir tmp_index_dir(m_index_dir); -- cgit From c4316e81e64ad4ac63b0b50106b324a73abdc150 Mon Sep 17 00:00:00 2001 From: flow Date: Sun, 26 Jun 2022 14:17:15 -0300 Subject: change: make Mod a QObject used as a pointer Prevents problems when copying it around! Signed-off-by: flow --- launcher/modplatform/EnsureMetadataTask.cpp | 80 ++++++++++++++++------------- 1 file changed, 44 insertions(+), 36 deletions(-) (limited to 'launcher/modplatform/EnsureMetadataTask.cpp') diff --git a/launcher/modplatform/EnsureMetadataTask.cpp b/launcher/modplatform/EnsureMetadataTask.cpp index 19e44ce0..7c153511 100644 --- a/launcher/modplatform/EnsureMetadataTask.cpp +++ b/launcher/modplatform/EnsureMetadataTask.cpp @@ -19,7 +19,7 @@ static ModPlatform::ProviderCapabilities ProviderCaps; static ModrinthAPI modrinth_api; static FlameAPI flame_api; -EnsureMetadataTask::EnsureMetadataTask(Mod& mod, QDir dir, ModPlatform::Provider prov) : Task(nullptr), m_index_dir(dir), m_provider(prov) +EnsureMetadataTask::EnsureMetadataTask(Mod* mod, QDir dir, ModPlatform::Provider prov) : Task(nullptr), m_index_dir(dir), m_provider(prov) { auto hash = getHash(mod); if (hash.isEmpty()) @@ -28,11 +28,11 @@ EnsureMetadataTask::EnsureMetadataTask(Mod& mod, QDir dir, ModPlatform::Provider m_mods.insert(hash, mod); } -EnsureMetadataTask::EnsureMetadataTask(std::list& mods, QDir dir, ModPlatform::Provider prov) +EnsureMetadataTask::EnsureMetadataTask(std::list& mods, QDir dir, ModPlatform::Provider prov) : Task(nullptr), m_index_dir(dir), m_provider(prov) { - for (auto& mod : mods) { - if (!mod.valid()) { + for (auto* mod : mods) { + if (!mod->valid()) { emitFail(mod); continue; } @@ -47,14 +47,14 @@ EnsureMetadataTask::EnsureMetadataTask(std::list& mods, QDir dir, ModPlatfo } } -QString EnsureMetadataTask::getHash(Mod& mod) +QString EnsureMetadataTask::getHash(Mod* mod) { /* Here we create a mapping hash -> mod, because we need that relationship to parse the API routes */ QByteArray jar_data; try { - jar_data = FS::read(mod.fileinfo().absoluteFilePath()); + jar_data = FS::read(mod->fileinfo().absoluteFilePath()); } catch (FS::FileSystemException& e) { - qCritical() << QString("Failed to open / read JAR file of %1").arg(mod.name()); + qCritical() << QString("Failed to open / read JAR file of %1").arg(mod->name()); qCritical() << QString("Reason: ") << e.cause(); return {}; @@ -95,19 +95,19 @@ void EnsureMetadataTask::executeTask() { setStatus(tr("Checking if mods have metadata...")); - for (auto mod : m_mods) { - if (!mod.valid()) + for (auto* mod : m_mods) { + if (!mod->valid()) continue; // They already have the right metadata :o - if (mod.status() != ModStatus::NoMetadata && mod.metadata() && mod.metadata()->provider == m_provider) { - qDebug() << "Mod" << mod.name() << "already has metadata!"; + if (mod->status() != ModStatus::NoMetadata && mod->metadata() && mod->metadata()->provider == m_provider) { + qDebug() << "Mod" << mod->name() << "already has metadata!"; emitReady(mod); return; } // Folders don't have metadata - if (mod.type() == Mod::MOD_FOLDER) { + if (mod->type() == Mod::MOD_FOLDER) { emitReady(mod); return; } @@ -125,7 +125,7 @@ void EnsureMetadataTask::executeTask() } auto invalidade_leftover = [this] { - QMutableHashIterator mods_iter(m_mods); + QMutableHashIterator mods_iter(m_mods); while (mods_iter.hasNext()) { auto mod = mods_iter.next(); emitFail(mod.value()); @@ -170,23 +170,23 @@ void EnsureMetadataTask::executeTask() setStatus(tr("Requesting metadata information from %1...").arg(ProviderCaps.readableName(m_provider))); else if (!m_mods.empty()) setStatus(tr("Requesting metadata information from %1 for '%2'...") - .arg(ProviderCaps.readableName(m_provider), m_mods.begin().value().name())); + .arg(ProviderCaps.readableName(m_provider), m_mods.begin().value()->name())); m_current_task = version_task.get(); version_task->start(); } -void EnsureMetadataTask::emitReady(Mod& m) +void EnsureMetadataTask::emitReady(Mod* m) { - qDebug() << QString("Generated metadata for %1").arg(m.name()); + qDebug() << QString("Generated metadata for %1").arg(m->name()); emit metadataReady(m); m_mods.remove(getHash(m)); } -void EnsureMetadataTask::emitFail(Mod& m) +void EnsureMetadataTask::emitFail(Mod* m) { - qDebug() << QString("Failed to generate metadata for %1").arg(m.name()); + qDebug() << QString("Failed to generate metadata for %1").arg(m->name()); emit metadataFailed(m); m_mods.remove(getHash(m)); @@ -224,8 +224,8 @@ NetJob::Ptr EnsureMetadataTask::modrinthVersionsTask() try { auto entry = Json::requireObject(entries, hash); - setStatus(tr("Parsing API response from Modrinth for '%1'...").arg(mod.name())); - qDebug() << "Getting version for" << mod.name() << "from Modrinth"; + setStatus(tr("Parsing API response from Modrinth for '%1'...").arg(mod->name())); + qDebug() << "Getting version for" << mod->name() << "from Modrinth"; m_temp_versions.insert(hash, Modrinth::loadIndexedPackVersion(entry)); } catch (Json::JsonException& e) { @@ -284,17 +284,22 @@ NetJob::Ptr EnsureMetadataTask::modrinthProjectsTask() for (auto entry : entries) { auto entry_obj = Json::requireObject(entry); - auto entry_id = Json::requireString(entry_obj, "id"); - auto hash = addonIds.find(entry_id).value(); + ModPlatform::IndexedPack pack; + Modrinth::loadIndexedPack(pack, entry_obj); - auto mod = m_mods.find(hash).value(); + auto hash = addonIds.find(pack.addonId.toString()).value(); - try { - setStatus(tr("Parsing API response from Modrinth for '%1'...").arg(mod.name())); + auto mod_iter = m_mods.find(hash); + if (mod_iter == m_mods.end()) { + qWarning() << "Invalid project id from the API response."; + continue; + } - ModPlatform::IndexedPack pack; - Modrinth::loadIndexedPack(pack, entry_obj); + auto* mod = mod_iter.value(); + + try { + setStatus(tr("Parsing API response from Modrinth for '%1'...").arg(mod->name())); modrinthCallback(pack, m_temp_versions.find(hash).value(), mod); } catch (Json::JsonException& e) { @@ -365,7 +370,7 @@ NetJob::Ptr EnsureMetadataTask::flameVersionsTask() continue; } - setStatus(tr("Parsing API response from CurseForge for '%1'...").arg(mod->name())); + setStatus(tr("Parsing API response from CurseForge for '%1'...").arg((*mod)->name())); m_temp_versions.insert(fingerprint, FlameMod::loadIndexedPackVersion(file_obj)); } @@ -385,7 +390,10 @@ NetJob::Ptr EnsureMetadataTask::flameProjectsTask() for (auto const& hash : m_mods.keys()) { if (m_temp_versions.contains(hash)) { auto const& data = m_temp_versions.find(hash).value(); - addonIds.insert(data.addonId.toString(), hash); + + auto id_str = data.addonId.toString(); + if (!id_str.isEmpty()) + addonIds.insert(data.addonId.toString(), hash); } } @@ -429,7 +437,7 @@ NetJob::Ptr EnsureMetadataTask::flameProjectsTask() auto mod = m_mods.find(hash).value(); try { - setStatus(tr("Parsing API response from CurseForge for '%1'...").arg(mod.name())); + setStatus(tr("Parsing API response from CurseForge for '%1'...").arg(mod->name())); ModPlatform::IndexedPack pack; FlameMod::loadIndexedPack(pack, entry_obj); @@ -451,10 +459,10 @@ NetJob::Ptr EnsureMetadataTask::flameProjectsTask() return proj_task; } -void EnsureMetadataTask::modrinthCallback(ModPlatform::IndexedPack& pack, ModPlatform::IndexedVersion& ver, Mod& mod) +void EnsureMetadataTask::modrinthCallback(ModPlatform::IndexedPack& pack, ModPlatform::IndexedVersion& ver, Mod* mod) { // Prevent file name mismatch - ver.fileName = mod.fileinfo().fileName(); + ver.fileName = mod->fileinfo().fileName(); if (ver.fileName.endsWith(".disabled")) ver.fileName.chop(9); @@ -479,16 +487,16 @@ void EnsureMetadataTask::modrinthCallback(ModPlatform::IndexedPack& pack, ModPla return; } - mod.setMetadata(metadata); + mod->setMetadata(metadata); emitReady(mod); } -void EnsureMetadataTask::flameCallback(ModPlatform::IndexedPack& pack, ModPlatform::IndexedVersion& ver, Mod& mod) +void EnsureMetadataTask::flameCallback(ModPlatform::IndexedPack& pack, ModPlatform::IndexedVersion& ver, Mod* mod) { try { // Prevent file name mismatch - ver.fileName = mod.fileinfo().fileName(); + ver.fileName = mod->fileinfo().fileName(); if (ver.fileName.endsWith(".disabled")) ver.fileName.chop(9); @@ -513,7 +521,7 @@ void EnsureMetadataTask::flameCallback(ModPlatform::IndexedPack& pack, ModPlatfo return; } - mod.setMetadata(metadata); + mod->setMetadata(metadata); emitReady(mod); } catch (Json::JsonException& e) { -- cgit From de9e304236ac0c11dd2b6bfb8b6f55943349c0e9 Mon Sep 17 00:00:00 2001 From: flow Date: Sun, 10 Jul 2022 15:15:25 -0300 Subject: fix: std::list -> QList Qt6 removed Qlist::toStdList() :sob: Signed-off-by: flow --- launcher/modplatform/EnsureMetadataTask.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'launcher/modplatform/EnsureMetadataTask.cpp') diff --git a/launcher/modplatform/EnsureMetadataTask.cpp b/launcher/modplatform/EnsureMetadataTask.cpp index 7c153511..f0c1fa99 100644 --- a/launcher/modplatform/EnsureMetadataTask.cpp +++ b/launcher/modplatform/EnsureMetadataTask.cpp @@ -28,7 +28,7 @@ EnsureMetadataTask::EnsureMetadataTask(Mod* mod, QDir dir, ModPlatform::Provider m_mods.insert(hash, mod); } -EnsureMetadataTask::EnsureMetadataTask(std::list& mods, QDir dir, ModPlatform::Provider prov) +EnsureMetadataTask::EnsureMetadataTask(QList& mods, QDir dir, ModPlatform::Provider prov) : Task(nullptr), m_index_dir(dir), m_provider(prov) { for (auto* mod : mods) { @@ -323,7 +323,7 @@ NetJob::Ptr EnsureMetadataTask::flameVersionsTask() { auto* response = new QByteArray(); - std::list fingerprints; + QList fingerprints; for (auto& murmur : m_mods.keys()) { fingerprints.push_back(murmur.toUInt()); } -- cgit From 6bb8332b4b8efbb3f21f31d465fa907c3db0a7ce Mon Sep 17 00:00:00 2001 From: flow Date: Sun, 17 Jul 2022 11:43:12 -0300 Subject: fix: bogus returns in EnsureMetadataTask Signed-off-by: flow --- launcher/modplatform/EnsureMetadataTask.cpp | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) (limited to 'launcher/modplatform/EnsureMetadataTask.cpp') diff --git a/launcher/modplatform/EnsureMetadataTask.cpp b/launcher/modplatform/EnsureMetadataTask.cpp index f0c1fa99..60c54c4e 100644 --- a/launcher/modplatform/EnsureMetadataTask.cpp +++ b/launcher/modplatform/EnsureMetadataTask.cpp @@ -96,20 +96,22 @@ void EnsureMetadataTask::executeTask() setStatus(tr("Checking if mods have metadata...")); for (auto* mod : m_mods) { - if (!mod->valid()) + if (!mod->valid()) { + qDebug() << "Mod" << mod->name() << "is invalid!"; + emitFail(mod); continue; + } // They already have the right metadata :o if (mod->status() != ModStatus::NoMetadata && mod->metadata() && mod->metadata()->provider == m_provider) { qDebug() << "Mod" << mod->name() << "already has metadata!"; emitReady(mod); - return; + continue; } // Folders don't have metadata if (mod->type() == Mod::MOD_FOLDER) { emitReady(mod); - return; } } -- cgit