aboutsummaryrefslogtreecommitdiff
path: root/launcher/net
diff options
context:
space:
mode:
Diffstat (limited to 'launcher/net')
-rw-r--r--launcher/net/Download.cpp4
-rw-r--r--launcher/net/Download.h2
-rw-r--r--launcher/net/HttpMetaCache.cpp23
-rw-r--r--launcher/net/HttpMetaCache.h17
-rw-r--r--launcher/net/MetaCacheSink.cpp45
-rw-r--r--launcher/net/MetaCacheSink.h3
-rw-r--r--launcher/net/NetAction.h2
-rw-r--r--launcher/net/NetJob.cpp202
-rw-r--r--launcher/net/NetJob.h48
-rw-r--r--launcher/net/Upload.cpp2
10 files changed, 146 insertions, 202 deletions
diff --git a/launcher/net/Download.cpp b/launcher/net/Download.cpp
index e6a6adcc..fd3dbedc 100644
--- a/launcher/net/Download.cpp
+++ b/launcher/net/Download.cpp
@@ -60,7 +60,7 @@ auto Download::makeCached(QUrl url, MetaEntryPtr entry, Options options) -> Down
dl->m_url = url;
dl->m_options = options;
auto md5Node = new ChecksumValidator(QCryptographicHash::Md5);
- auto cachedNode = new MetaCacheSink(entry, md5Node);
+ auto cachedNode = new MetaCacheSink(entry, md5Node, options.testFlag(Option::MakeEternal));
dl->m_sink.reset(cachedNode);
return dl;
}
@@ -118,7 +118,7 @@ void Download::executeTask()
}
request.setHeader(QNetworkRequest::UserAgentHeader, APPLICATION->getUserAgent().toUtf8());
- if (APPLICATION->currentCapabilities() & Application::SupportsFlame
+ if (APPLICATION->capabilities() & Application::SupportsFlame
&& request.url().host().contains("api.curseforge.com")) {
request.setRawHeader("x-api-key", APPLICATION->getFlameAPIKey().toUtf8());
};
diff --git a/launcher/net/Download.h b/launcher/net/Download.h
index 1d264381..3faa5db5 100644
--- a/launcher/net/Download.h
+++ b/launcher/net/Download.h
@@ -49,7 +49,7 @@ class Download : public NetAction {
public:
using Ptr = shared_qobject_ptr<class Download>;
- enum class Option { NoOptions = 0, AcceptLocalFiles = 1 };
+ enum class Option { NoOptions = 0, AcceptLocalFiles = 1, MakeEternal = 2 };
Q_DECLARE_FLAGS(Options, Option)
protected:
diff --git a/launcher/net/HttpMetaCache.cpp b/launcher/net/HttpMetaCache.cpp
index 4d86c0b8..9606ddb6 100644
--- a/launcher/net/HttpMetaCache.cpp
+++ b/launcher/net/HttpMetaCache.cpp
@@ -121,6 +121,14 @@ auto HttpMetaCache::resolveEntry(QString base, QString resource_path, QString ex
SaveEventually();
}
+ // Get rid of old entries, to prevent cache problems
+ auto current_time = QDateTime::currentSecsSinceEpoch();
+ if (entry->isExpired(current_time - ( file_last_changed / 1000 ))) {
+ qWarning() << "Removing cache entry because of old age!";
+ selected_base.entry_list.remove(resource_path);
+ return staleEntry(base, resource_path);
+ }
+
// entry passed all the checks we cared about.
entry->basePath = getBasePath(base);
return entry;
@@ -221,6 +229,13 @@ void HttpMetaCache::Load()
foo->etag = Json::ensureString(element_obj, "etag");
foo->local_changed_timestamp = Json::ensureDouble(element_obj, "last_changed_timestamp");
foo->remote_changed_timestamp = Json::ensureString(element_obj, "remote_changed_timestamp");
+
+ foo->makeEternal(Json::ensureBoolean(element_obj, "eternal", false));
+ if (!foo->isEternal()) {
+ foo->current_age = Json::ensureDouble(element_obj, "current_age");
+ foo->max_age = Json::ensureDouble(element_obj, "max_age");
+ }
+
// presumed innocent until closer examination
foo->stale = false;
@@ -240,6 +255,8 @@ void HttpMetaCache::SaveNow()
if (m_index_file.isNull())
return;
+ qDebug() << "[HttpMetaCache]" << "Saving metacache with" << m_entries.size() << "entries";
+
QJsonObject toplevel;
Json::writeString(toplevel, "version", "1");
@@ -259,6 +276,12 @@ void HttpMetaCache::SaveNow()
entryObj.insert("last_changed_timestamp", QJsonValue(double(entry->local_changed_timestamp)));
if (!entry->remote_changed_timestamp.isEmpty())
entryObj.insert("remote_changed_timestamp", QJsonValue(entry->remote_changed_timestamp));
+ if (entry->isEternal()) {
+ entryObj.insert("eternal", true);
+ } else {
+ entryObj.insert("current_age", QJsonValue(double(entry->current_age)));
+ entryObj.insert("max_age", QJsonValue(double(entry->max_age)));
+ }
entriesArr.append(entryObj);
}
}
diff --git a/launcher/net/HttpMetaCache.h b/launcher/net/HttpMetaCache.h
index e944b3d5..c0b12318 100644
--- a/launcher/net/HttpMetaCache.h
+++ b/launcher/net/HttpMetaCache.h
@@ -64,14 +64,31 @@ class MetaEntry {
auto getMD5Sum() -> QString { return md5sum; }
void setMD5Sum(QString md5sum) { this->md5sum = md5sum; }
+ /* Whether the entry expires after some time (false) or not (true). */
+ void makeEternal(bool eternal) { is_eternal = eternal; }
+ [[nodiscard]] bool isEternal() const { return is_eternal; }
+
+ auto getCurrentAge() -> qint64 { return current_age; }
+ void setCurrentAge(qint64 age) { current_age = age; }
+
+ auto getMaximumAge() -> qint64 { return max_age; }
+ void setMaximumAge(qint64 age) { max_age = age; }
+
+ bool isExpired(qint64 offset) { return !is_eternal && (current_age >= max_age - offset); };
+
protected:
QString baseId;
QString basePath;
QString relativePath;
QString md5sum;
QString etag;
+
qint64 local_changed_timestamp = 0;
QString remote_changed_timestamp; // QString for now, RFC 2822 encoded time
+ qint64 current_age = 0;
+ qint64 max_age = 0;
+ bool is_eternal = false;
+
bool stale = true;
};
diff --git a/launcher/net/MetaCacheSink.cpp b/launcher/net/MetaCacheSink.cpp
index f86dd870..5ae53c1c 100644
--- a/launcher/net/MetaCacheSink.cpp
+++ b/launcher/net/MetaCacheSink.cpp
@@ -36,13 +36,18 @@
#include "MetaCacheSink.h"
#include <QFile>
#include <QFileInfo>
-#include "FileSystem.h"
#include "Application.h"
namespace Net {
-MetaCacheSink::MetaCacheSink(MetaEntryPtr entry, ChecksumValidator * md5sum)
- :Net::FileSink(entry->getFullPath()), m_entry(entry), m_md5Node(md5sum)
+/** Maximum time to hold a cache entry
+ * = 1 week in seconds
+ */
+#define MAX_TIME_TO_EXPIRE 1*7*24*60*60
+
+
+MetaCacheSink::MetaCacheSink(MetaEntryPtr entry, ChecksumValidator * md5sum, bool is_eternal)
+ :Net::FileSink(entry->getFullPath()), m_entry(entry), m_md5Node(md5sum), m_is_eternal(is_eternal)
{
addValidator(md5sum);
}
@@ -88,6 +93,40 @@ Task::State MetaCacheSink::finalizeCache(QNetworkReply & reply)
}
m_entry->setLocalChangedTimestamp(output_file_info.lastModified().toUTC().toMSecsSinceEpoch());
+
+ { // Cache lifetime
+ if (m_is_eternal) {
+ qDebug() << "[MetaCache] Adding eternal cache entry:" << m_entry->getFullPath();
+ m_entry->makeEternal(true);
+ } else if (reply.hasRawHeader("Cache-Control")) {
+ auto cache_control_header = reply.rawHeader("Cache-Control");
+ // qDebug() << "[MetaCache] Parsing 'Cache-Control' header with" << cache_control_header;
+
+ QRegularExpression max_age_expr("max-age=([0-9]+)");
+ qint64 max_age = max_age_expr.match(cache_control_header).captured(1).toLongLong();
+ m_entry->setMaximumAge(max_age);
+
+ } else if (reply.hasRawHeader("Expires")) {
+ auto expires_header = reply.rawHeader("Expires");
+ // qDebug() << "[MetaCache] Parsing 'Expires' header with" << expires_header;
+
+ qint64 max_age = QDateTime::fromString(expires_header).toSecsSinceEpoch() - QDateTime::currentSecsSinceEpoch();
+ m_entry->setMaximumAge(max_age);
+ } else {
+ m_entry->setMaximumAge(MAX_TIME_TO_EXPIRE);
+ }
+
+ if (reply.hasRawHeader("Age")) {
+ auto age_header = reply.rawHeader("Age");
+ // qDebug() << "[MetaCache] Parsing 'Age' header with" << age_header;
+
+ qint64 current_age = age_header.toLongLong();
+ m_entry->setCurrentAge(current_age);
+ } else {
+ m_entry->setCurrentAge(0);
+ }
+ }
+
m_entry->setStale(false);
APPLICATION->metacache()->updateEntry(m_entry);
diff --git a/launcher/net/MetaCacheSink.h b/launcher/net/MetaCacheSink.h
index c9f7edfe..f5948085 100644
--- a/launcher/net/MetaCacheSink.h
+++ b/launcher/net/MetaCacheSink.h
@@ -42,7 +42,7 @@
namespace Net {
class MetaCacheSink : public FileSink {
public:
- MetaCacheSink(MetaEntryPtr entry, ChecksumValidator* md5sum);
+ MetaCacheSink(MetaEntryPtr entry, ChecksumValidator* md5sum, bool is_eternal = false);
virtual ~MetaCacheSink() = default;
auto hasLocalData() -> bool override;
@@ -54,5 +54,6 @@ class MetaCacheSink : public FileSink {
private:
MetaEntryPtr m_entry;
ChecksumValidator* m_md5Node;
+ bool m_is_eternal;
};
} // namespace Net
diff --git a/launcher/net/NetAction.h b/launcher/net/NetAction.h
index 729d4132..d9c4fadc 100644
--- a/launcher/net/NetAction.h
+++ b/launcher/net/NetAction.h
@@ -54,6 +54,8 @@ class NetAction : public Task {
QUrl url() { return m_url; }
auto index() -> int { return m_index_within_job; }
+ void setNetwork(shared_qobject_ptr<QNetworkAccessManager> network) { m_network = network; }
+
protected slots:
virtual void downloadProgress(qint64 bytesReceived, qint64 bytesTotal) = 0;
virtual void downloadError(QNetworkReply::NetworkError error) = 0;
diff --git a/launcher/net/NetJob.cpp b/launcher/net/NetJob.cpp
index bab35fa5..20d75976 100644
--- a/launcher/net/NetJob.cpp
+++ b/launcher/net/NetJob.cpp
@@ -35,204 +35,90 @@
*/
#include "NetJob.h"
-#include "Download.h"
auto NetJob::addNetAction(NetAction::Ptr action) -> bool
{
- action->m_index_within_job = m_downloads.size();
- m_downloads.append(action);
- part_info pi;
- m_parts_progress.append(pi);
-
- partProgress(m_parts_progress.count() - 1, action->getProgress(), action->getTotalProgress());
-
- if (action->isRunning()) {
- connect(action.get(), &NetAction::succeeded, [this, action]{ partSucceeded(action->index()); });
- connect(action.get(), &NetAction::failed, [this, action](QString){ partFailed(action->index()); });
- connect(action.get(), &NetAction::aborted, [this, action](){ partAborted(action->index()); });
- connect(action.get(), &NetAction::progress, [this, action](qint64 done, qint64 total) { partProgress(action->index(), done, total); });
- connect(action.get(), &NetAction::status, this, &NetJob::status);
- } else {
- m_todo.append(m_parts_progress.size() - 1);
- }
+ action->m_index_within_job = m_queue.size();
+ m_queue.append(action);
+
+ action->setNetwork(m_network);
return true;
}
+void NetJob::startNext()
+{
+ if (m_queue.isEmpty() && m_doing.isEmpty()) {
+ // We're finished, check for failures and retry if we can (up to 3 times)
+ if (!m_failed.isEmpty() && m_try < 3) {
+ m_try += 1;
+ while (!m_failed.isEmpty())
+ m_queue.enqueue(m_failed.take(*m_failed.keyBegin()));
+ }
+ }
+
+ ConcurrentTask::startNext();
+}
+
+auto NetJob::size() const -> int
+{
+ return m_queue.size() + m_doing.size() + m_done.size();
+}
+
auto NetJob::canAbort() const -> bool
{
bool canFullyAbort = true;
// can abort the downloads on the queue?
- for (auto index : m_todo) {
- auto part = m_downloads[index];
+ for (auto part : m_queue)
canFullyAbort &= part->canAbort();
- }
+
// can abort the active downloads?
- for (auto index : m_doing) {
- auto part = m_downloads[index];
+ for (auto part : m_doing)
canFullyAbort &= part->canAbort();
- }
return canFullyAbort;
}
-void NetJob::executeTask()
-{
- // hack that delays early failures so they can be caught easier
- QMetaObject::invokeMethod(this, "startMoreParts", Qt::QueuedConnection);
-}
-
-auto NetJob::getFailedFiles() -> QStringList
-{
- QStringList failed;
- for (auto index : m_failed) {
- failed.push_back(m_downloads[index]->url().toString());
- }
- failed.sort();
- return failed;
-}
-
auto NetJob::abort() -> bool
{
bool fullyAborted = true;
// fail all downloads on the queue
-#if QT_VERSION >= QT_VERSION_CHECK(5, 14, 0)
- QSet<int> todoSet(m_todo.begin(), m_todo.end());
- m_failed.unite(todoSet);
-#else
- m_failed.unite(m_todo.toSet());
-#endif
- m_todo.clear();
+ for (auto task : m_queue)
+ m_failed.insert(task.get(), task);
+ m_queue.clear();
// abort active downloads
auto toKill = m_doing.values();
- for (auto index : toKill) {
- auto part = m_downloads[index];
+ for (auto part : toKill) {
fullyAborted &= part->abort();
}
return fullyAborted;
}
-void NetJob::partSucceeded(int index)
-{
- // do progress. all slots are 1 in size at least
- auto& slot = m_parts_progress[index];
- partProgress(index, slot.total_progress, slot.total_progress);
-
- m_doing.remove(index);
- m_done.insert(index);
- m_downloads[index].get()->disconnect(this);
-
- startMoreParts();
-}
-
-void NetJob::partFailed(int index)
+auto NetJob::getFailedActions() -> QList<NetAction*>
{
- m_doing.remove(index);
-
- auto& slot = m_parts_progress[index];
- // Can try 3 times before failing by definitive
- if (slot.failures == 3) {
- m_failed.insert(index);
- } else {
- slot.failures++;
- m_todo.enqueue(index);
+ QList<NetAction*> failed;
+ for (auto index : m_failed) {
+ failed.push_back(dynamic_cast<NetAction*>(index.get()));
}
-
- m_downloads[index].get()->disconnect(this);
-
- startMoreParts();
-}
-
-void NetJob::partAborted(int index)
-{
- m_aborted = true;
-
- m_doing.remove(index);
- m_failed.insert(index);
- m_downloads[index].get()->disconnect(this);
-
- startMoreParts();
+ return failed;
}
-void NetJob::partProgress(int index, qint64 bytesReceived, qint64 bytesTotal)
+auto NetJob::getFailedFiles() -> QList<QString>
{
- auto& slot = m_parts_progress[index];
- slot.current_progress = bytesReceived;
- slot.total_progress = bytesTotal;
-
- int done = m_done.size();
- int doing = m_doing.size();
- int all = m_parts_progress.size();
-
- qint64 bytesAll = 0;
- qint64 bytesTotalAll = 0;
- for (auto& partIdx : m_doing) {
- auto part = m_parts_progress[partIdx];
- // do not count parts with unknown/nonsensical total size
- if (part.total_progress <= 0) {
- continue;
- }
- bytesAll += part.current_progress;
- bytesTotalAll += part.total_progress;
- }
-
- qint64 inprogress = (bytesTotalAll == 0) ? 0 : (bytesAll * 1000) / bytesTotalAll;
- auto current = done * 1000 + doing * inprogress;
- auto current_total = all * 1000;
- // HACK: make sure it never jumps backwards.
- // FAIL: This breaks if the size is not known (or is it something else?) and jumps to 1000, so if it is 1000 reset it to inprogress
- if (m_current_progress == 1000) {
- m_current_progress = inprogress;
- }
- if (m_current_progress > current) {
- current = m_current_progress;
+ QList<QString> failed;
+ for (auto index : m_failed) {
+ failed.append(static_cast<NetAction*>(index.get())->url().toString());
}
- m_current_progress = current;
- setProgress(current, current_total);
+ return failed;
}
-void NetJob::startMoreParts()
+void NetJob::updateState()
{
- if (!isRunning()) {
- // this actually makes sense. You can put running m_downloads into a NetJob and then not start it until much later.
- return;
- }
-
- // OK. We are actively processing tasks, proceed.
- // Check for final conditions if there's nothing in the queue.
- if (!m_todo.size()) {
- if (!m_doing.size()) {
- if (!m_failed.size()) {
- emitSucceeded();
- } else if (m_aborted) {
- emitAborted();
- } else {
- emitFailed(tr("Job '%1' failed to process:\n%2").arg(objectName()).arg(getFailedFiles().join("\n")));
- }
- }
- return;
- }
-
- // There's work to do, try to start more parts, to a maximum of 6 concurrent ones.
- while (m_doing.size() < 6) {
- if (m_todo.size() == 0)
- return;
- int doThis = m_todo.dequeue();
- m_doing.insert(doThis);
-
- auto part = m_downloads[doThis];
-
- // connect signals :D
- connect(part.get(), &NetAction::succeeded, this, [this, part]{ partSucceeded(part->index()); });
- connect(part.get(), &NetAction::failed, this, [this, part](QString){ partFailed(part->index()); });
- connect(part.get(), &NetAction::aborted, this, [this, part]{ partAborted(part->index()); });
- connect(part.get(), &NetAction::progress, this, [this, part](qint64 done, qint64 total) { partProgress(part->index(), done, total); });
- connect(part.get(), &NetAction::status, this, &NetJob::status);
-
- part->startAction(m_network);
- }
+ emit progress(m_done.count(), m_total_size);
+ setStatus(tr("Executing %1 task(s) (%2 out of %3 are done)")
+ .arg(QString::number(m_doing.count()), QString::number(m_done.count()), QString::number(m_total_size)));
}
diff --git a/launcher/net/NetJob.h b/launcher/net/NetJob.h
index 63c1cf51..cd5d5e48 100644
--- a/launcher/net/NetJob.h
+++ b/launcher/net/NetJob.h
@@ -39,64 +39,40 @@
#include <QObject>
#include "NetAction.h"
-#include "tasks/Task.h"
+#include "tasks/ConcurrentTask.h"
// Those are included so that they are also included by anyone using NetJob
#include "net/Download.h"
#include "net/HttpMetaCache.h"
-class NetJob : public Task {
+class NetJob : public ConcurrentTask {
Q_OBJECT
public:
using Ptr = shared_qobject_ptr<NetJob>;
- explicit NetJob(QString job_name, shared_qobject_ptr<QNetworkAccessManager> network) : Task(), m_network(network)
- {
- setObjectName(job_name);
- }
- virtual ~NetJob() = default;
+ explicit NetJob(QString job_name, shared_qobject_ptr<QNetworkAccessManager> network) : ConcurrentTask(nullptr, job_name), m_network(network) {}
+ ~NetJob() override = default;
- void executeTask() override;
+ void startNext() override;
- auto canAbort() const -> bool override;
+ auto size() const -> int;
+ auto canAbort() const -> bool override;
auto addNetAction(NetAction::Ptr action) -> bool;
- auto operator[](int index) -> NetAction::Ptr { return m_downloads[index]; }
- auto at(int index) -> const NetAction::Ptr { return m_downloads.at(index); }
- auto size() const -> int { return m_downloads.size(); }
- auto first() -> NetAction::Ptr { return m_downloads.size() != 0 ? m_downloads[0] : NetAction::Ptr{}; }
-
- auto getFailedFiles() -> QStringList;
+ auto getFailedActions() -> QList<NetAction*>;
+ auto getFailedFiles() -> QList<QString>;
public slots:
// Qt can't handle auto at the start for some reason?
bool abort() override;
- private slots:
- void startMoreParts();
-
- void partProgress(int index, qint64 bytesReceived, qint64 bytesTotal);
- void partSucceeded(int index);
- void partFailed(int index);
- void partAborted(int index);
+ protected:
+ void updateState() override;
private:
shared_qobject_ptr<QNetworkAccessManager> m_network;
- struct part_info {
- qint64 current_progress = 0;
- qint64 total_progress = 1;
- int failures = 0;
- };
-
- QList<NetAction::Ptr> m_downloads;
- QList<part_info> m_parts_progress;
- QQueue<int> m_todo;
- QSet<int> m_doing;
- QSet<int> m_done;
- QSet<int> m_failed;
- qint64 m_current_progress = 0;
- bool m_aborted = false;
+ int m_try = 1;
};
diff --git a/launcher/net/Upload.cpp b/launcher/net/Upload.cpp
index cfda4b4e..f3b19022 100644
--- a/launcher/net/Upload.cpp
+++ b/launcher/net/Upload.cpp
@@ -216,7 +216,7 @@ namespace Net {
}
request.setHeader(QNetworkRequest::UserAgentHeader, APPLICATION->getUserAgent().toUtf8());
- if (APPLICATION->currentCapabilities() & Application::SupportsFlame
+ if (APPLICATION->capabilities() & Application::SupportsFlame
&& request.url().host().contains("api.curseforge.com")) {
request.setRawHeader("x-api-key", APPLICATION->getFlameAPIKey().toUtf8());
}
generated by cgit (git 2.46.0) at 2025-02-10 18:11:46 +0000