diff options
| -rw-r--r-- | CMakeLists.txt | 7 | ||||
| -rw-r--r-- | MultiMC.cpp | 21 | ||||
| -rw-r--r-- | MultiMC.h | 8 | ||||
| -rw-r--r-- | logic/LegacyUpdate.cpp | 4 | ||||
| -rw-r--r-- | logic/OneSixAssets.cpp | 29 | ||||
| -rw-r--r-- | logic/OneSixUpdate.cpp | 20 | ||||
| -rw-r--r-- | logic/net/ByteArrayDownload.cpp | 62 | ||||
| -rw-r--r-- | logic/net/ByteArrayDownload.h | 24 | ||||
| -rw-r--r-- | logic/net/CacheDownload.cpp | 122 | ||||
| -rw-r--r-- | logic/net/CacheDownload.h | 34 | ||||
| -rw-r--r-- | logic/net/Download.h | 54 | ||||
| -rw-r--r-- | logic/net/DownloadJob.cpp | 146 | ||||
| -rw-r--r-- | logic/net/DownloadJob.h | 95 | ||||
| -rw-r--r-- | logic/net/FileDownload.cpp | 111 | ||||
| -rw-r--r-- | logic/net/FileDownload.h | 35 | ||||
| -rw-r--r-- | logic/net/HttpMetaCache.cpp | 157 | ||||
| -rw-r--r-- | logic/net/HttpMetaCache.h | 33 |
17 files changed, 693 insertions, 269 deletions
diff --git a/CMakeLists.txt b/CMakeLists.txt index 27f8eaba..cbc3d842 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -184,6 +184,10 @@ logic/ModList.h logic/InstanceLauncher.h # network stuffs +logic/net/Download.h +logic/net/FileDownload.h +logic/net/ByteArrayDownload.h +logic/net/CacheDownload.h logic/net/DownloadJob.h logic/net/HttpMetaCache.h @@ -252,6 +256,9 @@ logic/ModList.cpp logic/InstanceLauncher.cpp # network stuffs - to be moved into a depend lib ~_~ +logic/net/FileDownload.cpp +logic/net/ByteArrayDownload.cpp +logic/net/CacheDownload.cpp logic/net/DownloadJob.cpp logic/net/HttpMetaCache.cpp diff --git a/MultiMC.cpp b/MultiMC.cpp index 88eb3e62..f212d830 100644 --- a/MultiMC.cpp +++ b/MultiMC.cpp @@ -8,6 +8,7 @@ #include "logic/lists/InstanceList.h" #include "logic/lists/IconList.h" #include "logic/InstanceLauncher.h" +#include "logic/net/HttpMetaCache.h" #include "pathutils.h" @@ -112,11 +113,16 @@ MultiMC::MultiMC ( int& argc, char** argv ) // load settings initGlobalSettings(); + // and instances m_instances = new InstanceList(m_settings->get("InstanceDir").toString(),this); std::cout << "Loading Instances..." << std::endl; m_instances->loadList(); - // network manager + + // init the http meta cache + initHttpMetaCache(); + + // create the global network manager m_qnam = new QNetworkAccessManager(this); // Register meta types. @@ -137,11 +143,12 @@ MultiMC::MultiMC ( int& argc, char** argv ) MultiMC::~MultiMC() { delete m_settings; + delete m_metacache; } void MultiMC::initGlobalSettings() { - m_settings = new INISettingsObject(applicationDirPath() + "/multimc.cfg", this); + m_settings = new INISettingsObject("multimc.cfg", this); // Updates m_settings->registerSetting(new Setting("UseDevBuilds", false)); m_settings->registerSetting(new Setting("AutoUpdate", true)); @@ -189,6 +196,16 @@ void MultiMC::initGlobalSettings() m_settings->registerSetting(new Setting("TheCat", false)); } +void MultiMC::initHttpMetaCache() +{ + m_metacache = new HttpMetaCache("metacache"); + m_metacache->addBase("assets", QDir("assets").absolutePath()); + m_metacache->addBase("versions", QDir("versions").absolutePath()); + m_metacache->addBase("libraries", QDir("libraries").absolutePath()); + m_metacache->Load(); +} + + IconList* MultiMC::icons() { if ( !m_icons ) @@ -4,6 +4,7 @@ #include "MultiMCVersion.h" #include "config.h" +class HttpMetaCache; class SettingsObject; class InstanceList; class IconList; @@ -55,14 +56,21 @@ public: { return m_qnam; } + + HttpMetaCache * metacache() + { + return m_metacache; + } private: void initGlobalSettings(); + void initHttpMetaCache(); private: SettingsObject * m_settings = nullptr; InstanceList * m_instances = nullptr; IconList * m_icons = nullptr; QNetworkAccessManager * m_qnam = nullptr; + HttpMetaCache * m_metacache = nullptr; Status m_status = MultiMC::Failed; MultiMCVersion m_version = {VERSION_MAJOR, VERSION_MINOR, VERSION_REVISION, VERSION_BUILD}; };
\ No newline at end of file diff --git a/logic/LegacyUpdate.cpp b/logic/LegacyUpdate.cpp index 626ad1e0..c75a0011 100644 --- a/logic/LegacyUpdate.cpp +++ b/logic/LegacyUpdate.cpp @@ -227,7 +227,9 @@ void LegacyUpdate::jarStart() QString intended_version_id = inst->intendedVersionId(); urlstr += intended_version_id + "/" + intended_version_id + ".jar"; - legacyDownloadJob.reset(new DownloadJob(QUrl(urlstr), inst->defaultBaseJar())); + auto dljob = new DownloadJob("Minecraft.jar for version " + intended_version_id); + dljob->add(QUrl(urlstr), inst->defaultBaseJar()); + legacyDownloadJob.reset(); connect(legacyDownloadJob.data(), SIGNAL(finished()), SLOT(jarFinished())); connect(legacyDownloadJob.data(), SIGNAL(failed()), SLOT(jarFailed())); connect(legacyDownloadJob.data(), SIGNAL(progress(qint64,qint64)), SLOT(updateDownloadProgress(qint64,qint64))); diff --git a/logic/OneSixAssets.cpp b/logic/OneSixAssets.cpp index c65ee607..5bdd29d7 100644 --- a/logic/OneSixAssets.cpp +++ b/logic/OneSixAssets.cpp @@ -3,6 +3,8 @@ #include <QtXml/QtXml> #include "OneSixAssets.h" #include "net/DownloadJob.h" +#include "net/HttpMetaCache.h" +#include "MultiMC.h" inline QDomElement getDomElementByTagName(QDomElement parent, QString tagname) { @@ -65,7 +67,7 @@ void OneSixAssets::fetchXMLFinished() nuke_whitelist.clear(); auto firstJob = index_job->first(); - QByteArray ba = firstJob->m_data; + QByteArray ba = firstJob.dynamicCast<ByteArrayDownload>()->m_data; QString xmlErrorMsg; QDomDocument doc; @@ -76,10 +78,12 @@ void OneSixAssets::fetchXMLFinished() //QRegExp etag_match(".*([a-f0-9]{32}).*"); QDomNodeList contents = doc.elementsByTagName ( "Contents" ); - DownloadJob *job = new DownloadJob(); + DownloadJob *job = new DownloadJob("Assets"); connect ( job, SIGNAL(succeeded()), SLOT(downloadFinished()) ); connect ( job, SIGNAL(failed()), SIGNAL(failed()) ); + auto metacache = MMC->metacache(); + for ( int i = 0; i < contents.length(); i++ ) { QDomElement element = contents.at ( i ).toElement(); @@ -104,22 +108,12 @@ void OneSixAssets::fetchXMLFinished() if ( sizeStr == "0" ) continue; - QString filename = fprefix + keyStr; - QFile check_file ( filename ); - QString client_etag = "nonsense"; - // if there already is a file and md5 checking is in effect and it can be opened - if ( check_file.exists() && check_file.open ( QIODevice::ReadOnly ) ) - { - // check the md5 against the expected one - client_etag = QCryptographicHash::hash ( check_file.readAll(), QCryptographicHash::Md5 ).toHex().constData(); - check_file.close(); - } - - QString trimmedEtag = etagStr.remove ( '"' ); nuke_whitelist.append ( keyStr ); - if(trimmedEtag != client_etag) + + auto entry = metacache->resolveEntry("assets", keyStr, etagStr); + if(entry->stale) { - job->add ( QUrl ( prefix + keyStr ), filename ); + job->add(QUrl(prefix + keyStr), entry); } } if(job->size()) @@ -135,7 +129,8 @@ void OneSixAssets::fetchXMLFinished() } void OneSixAssets::start() { - DownloadJob * job = new DownloadJob(QUrl ( "http://s3.amazonaws.com/Minecraft.Resources/" )); + auto job = new DownloadJob("Assets index"); + job->add(QUrl ( "http://s3.amazonaws.com/Minecraft.Resources/" )); connect ( job, SIGNAL(succeeded()), SLOT ( fetchXMLFinished() ) ); index_job.reset ( job ); job->start(); diff --git a/logic/OneSixUpdate.cpp b/logic/OneSixUpdate.cpp index 428d6ef7..ce71bde0 100644 --- a/logic/OneSixUpdate.cpp +++ b/logic/OneSixUpdate.cpp @@ -12,7 +12,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - +#include "MultiMC.h" #include "OneSixUpdate.h" #include <QtNetwork> @@ -72,7 +72,9 @@ void OneSixUpdate::versionFileStart() QString urlstr("http://s3.amazonaws.com/Minecraft.Download/versions/"); urlstr += targetVersion->descriptor + "/" + targetVersion->descriptor + ".json"; - specificVersionDownloadJob.reset(new DownloadJob(QUrl(urlstr))); + auto job = new DownloadJob("Version index"); + job->add(QUrl(urlstr)); + specificVersionDownloadJob.reset(job); connect(specificVersionDownloadJob.data(), SIGNAL(succeeded()), SLOT(versionFileFinished())); connect(specificVersionDownloadJob.data(), SIGNAL(failed()), SLOT(versionFileFailed())); connect(specificVersionDownloadJob.data(), SIGNAL(progress(qint64,qint64)), SLOT(updateDownloadProgress(qint64,qint64))); @@ -92,7 +94,7 @@ void OneSixUpdate::versionFileFinished() // FIXME: detect errors here, download to a temp file, swap QFile vfile1 (version1); vfile1.open(QIODevice::Truncate | QIODevice::WriteOnly ); - vfile1.write(DlJob->m_data); + vfile1.write(DlJob.dynamicCast<ByteArrayDownload>()->m_data); vfile1.close(); } @@ -134,16 +136,22 @@ void OneSixUpdate::jarlibStart() QString targetstr ("versions/"); targetstr += version->id + "/" + version->id + ".jar"; - jarlibDownloadJob.reset(new DownloadJob(QUrl(urlstr), targetstr)); + auto job = new DownloadJob("Libraries for instance " + inst->name()); + job->add(QUrl(urlstr), targetstr); + jarlibDownloadJob.reset(job); auto libs = version->getActiveNativeLibs(); libs.append(version->getActiveNormalLibs()); + auto metacache = MMC->metacache(); for(auto lib: libs) { QString download_path = lib->downloadPath(); - QString storage_path = "libraries/" + lib->storagePath(); - jarlibDownloadJob->add(download_path, storage_path); + auto entry = metacache->resolveEntry("libraries", lib->storagePath()); + if(entry->stale) + { + jarlibDownloadJob->add(download_path, entry); + } } connect(jarlibDownloadJob.data(), SIGNAL(succeeded()), SLOT(jarlibFinished())); connect(jarlibDownloadJob.data(), SIGNAL(failed()), SLOT(jarlibFailed())); diff --git a/logic/net/ByteArrayDownload.cpp b/logic/net/ByteArrayDownload.cpp new file mode 100644 index 00000000..6ae3f121 --- /dev/null +++ b/logic/net/ByteArrayDownload.cpp @@ -0,0 +1,62 @@ +#include "ByteArrayDownload.h" +#include "MultiMC.h" +#include <QDebug> + +ByteArrayDownload::ByteArrayDownload ( QUrl url ) + : Download() +{ + m_url = url; + m_status = Job_NotStarted; +} + +void ByteArrayDownload::start() +{ + qDebug() << "Downloading " << m_url.toString(); + QNetworkRequest request ( m_url ); + auto worker = MMC->qnam(); + QNetworkReply * rep = worker->get ( request ); + + m_reply = QSharedPointer<QNetworkReply> ( rep, &QObject::deleteLater ); + connect ( rep, SIGNAL ( downloadProgress ( qint64,qint64 ) ), SLOT ( downloadProgress ( qint64,qint64 ) ) ); + connect ( rep, SIGNAL ( finished() ), SLOT ( downloadFinished() ) ); + connect ( rep, SIGNAL ( error ( QNetworkReply::NetworkError ) ), SLOT ( downloadError ( QNetworkReply::NetworkError ) ) ); + connect ( rep, SIGNAL ( readyRead() ), SLOT ( downloadReadyRead() ) ); +} + +void ByteArrayDownload::downloadProgress ( qint64 bytesReceived, qint64 bytesTotal ) +{ + emit progress (index_within_job, bytesReceived, bytesTotal ); +} + +void ByteArrayDownload::downloadError ( QNetworkReply::NetworkError error ) +{ + // error happened during download. + // TODO: log the reason why + m_status = Job_Failed; +} + +void ByteArrayDownload::downloadFinished() +{ + // if the download succeeded + if ( m_status != Job_Failed ) + { + // nothing went wrong... + m_status = Job_Finished; + m_data = m_reply->readAll(); + m_reply.clear(); + emit succeeded(index_within_job); + return; + } + // else the download failed + else + { + m_reply.clear(); + emit failed(index_within_job); + return; + } +} + +void ByteArrayDownload::downloadReadyRead() +{ + // ~_~ +} diff --git a/logic/net/ByteArrayDownload.h b/logic/net/ByteArrayDownload.h new file mode 100644 index 00000000..428b21db --- /dev/null +++ b/logic/net/ByteArrayDownload.h @@ -0,0 +1,24 @@ +#pragma once +#include "Download.h" + +class ByteArrayDownload: public Download +{ + Q_OBJECT +public: + ByteArrayDownload(QUrl url); + +public: + /// if not saving to file, downloaded data is placed here + QByteArray m_data; + +public slots: + virtual void start(); + +protected slots: + void downloadProgress(qint64 bytesReceived, qint64 bytesTotal); + void downloadError(QNetworkReply::NetworkError error); + void downloadFinished(); + void downloadReadyRead(); +}; + +typedef QSharedPointer<ByteArrayDownload> ByteArrayDownloadPtr; diff --git a/logic/net/CacheDownload.cpp b/logic/net/CacheDownload.cpp new file mode 100644 index 00000000..c0074574 --- /dev/null +++ b/logic/net/CacheDownload.cpp @@ -0,0 +1,122 @@ +#include "MultiMC.h" +#include "CacheDownload.h" +#include <pathutils.h> + +#include <QCryptographicHash> +#include <QFileInfo> +#include <QDateTime> +#include <QDebug> + +CacheDownload::CacheDownload (QUrl url, MetaEntryPtr entry ) + :Download(), md5sum(QCryptographicHash::Md5) +{ + m_url = url; + m_entry = entry; + m_target_path = entry->getFullPath(); + m_status = Job_NotStarted; + m_opened_for_saving = false; +} + +void CacheDownload::start() +{ + if(!m_entry->stale) + { + emit succeeded(index_within_job); + return; + } + m_output_file.setFileName(m_target_path); + // if there already is a file and md5 checking is in effect and it can be opened + if(!ensureFilePathExists(m_target_path)) + { + emit failed(index_within_job); + return; + } + qDebug() << "Downloading " << m_url.toString(); + QNetworkRequest request ( m_url ); + request.setRawHeader(QString("If-None-Match").toLatin1(), m_entry->etag.toLatin1()); + + auto worker = MMC->qnam(); + QNetworkReply * rep = worker->get ( request ); + + m_reply = QSharedPointer<QNetworkReply> ( rep, &QObject::deleteLater ); + connect ( rep, SIGNAL ( downloadProgress ( qint64,qint64 ) ), SLOT ( downloadProgress ( qint64,qint64 ) ) ); + connect ( rep, SIGNAL ( finished() ), SLOT ( downloadFinished() ) ); + connect ( rep, SIGNAL ( error ( QNetworkReply::NetworkError ) ), SLOT ( downloadError ( QNetworkReply::NetworkError ) ) ); + connect ( rep, SIGNAL ( readyRead() ), SLOT ( downloadReadyRead() ) ); +} + +void CacheDownload::downloadProgress ( qint64 bytesReceived, qint64 bytesTotal ) +{ + emit progress (index_within_job, bytesReceived, bytesTotal ); +} + +void CacheDownload::downloadError ( QNetworkReply::NetworkError error ) +{ + // error happened during download. + // TODO: log the reason why + m_status = Job_Failed; +} +void CacheDownload::downloadFinished() +{ + // if the download succeeded + if ( m_status != Job_Failed ) + { + + // nothing went wrong... + m_status = Job_Finished; + if(m_opened_for_saving) + { + // save the data to the downloadable if we aren't saving to file + m_output_file.close(); + m_entry->md5sum = md5sum.result().toHex().constData(); + } + else + { + if ( m_output_file.open ( QIODevice::ReadOnly ) ) + { + m_entry->md5sum = QCryptographicHash::hash ( m_output_file.readAll(), QCryptographicHash::Md5 ).toHex().constData(); + m_output_file.close(); + } + } + QFileInfo output_file_info(m_target_path); + + + m_entry->etag = m_reply->rawHeader("ETag").constData(); + m_entry->last_changed_timestamp = output_file_info.lastModified().toUTC().toMSecsSinceEpoch(); + m_entry->stale = false; + MMC->metacache()->updateEntry(m_entry); + + m_reply.clear(); + emit succeeded(index_within_job); + return; + } + // else the download failed + else + { + m_output_file.close(); + m_output_file.remove(); + m_reply.clear(); + emit failed(index_within_job); + return; + } +} + +void CacheDownload::downloadReadyRead() +{ + if(!m_opened_for_saving) + { + if ( !m_output_file.open ( QIODevice::WriteOnly ) ) + { + /* + * Can't open the file... the job failed + */ + m_reply->abort(); + emit failed(index_within_job); + return; + } + m_opened_for_saving = true; + } + QByteArray ba = m_reply->readAll(); + md5sum.addData(ba); + m_output_file.write ( ba ); +} diff --git a/logic/net/CacheDownload.h b/logic/net/CacheDownload.h new file mode 100644 index 00000000..f95dabd5 --- /dev/null +++ b/logic/net/CacheDownload.h @@ -0,0 +1,34 @@ +#pragma once + +#include "Download.h" +#include "HttpMetaCache.h" +#include <QFile> +#include <qcryptographichash.h> + +class CacheDownload : public Download +{ + Q_OBJECT +public: + MetaEntryPtr m_entry; + /// is the saving file already open? + bool m_opened_for_saving; + /// if saving to file, use the one specified in this string + QString m_target_path; + /// this is the output file, if any + QFile m_output_file; + /// the hash-as-you-download + QCryptographicHash md5sum; +public: + explicit CacheDownload(QUrl url, MetaEntryPtr entry); + +protected slots: + virtual void downloadProgress(qint64 bytesReceived, qint64 bytesTotal); + virtual void downloadError(QNetworkReply::NetworkError error); + virtual void downloadFinished(); + virtual void downloadReadyRead(); + +public slots: + virtual void start(); +}; + +typedef QSharedPointer<CacheDownload> CacheDownloadPtr; diff --git a/logic/net/Download.h b/logic/net/Download.h new file mode 100644 index 00000000..91f09dec --- /dev/null +++ b/logic/net/Download.h @@ -0,0 +1,54 @@ +#pragma once + +#include <QObject> +#include <QUrl> +#include <QSharedPointer> +#include <QNetworkReply> + + +enum JobStatus +{ + Job_NotStarted, + Job_InProgress, + Job_Finished, + Job_Failed +}; + +class Download : public QObject +{ + Q_OBJECT +protected: + explicit Download(): QObject(0){}; +public: + virtual ~Download() {}; + +public: + /// the network reply + QSharedPointer<QNetworkReply> m_reply; + + /// source URL + QUrl m_url; + + /// The file's status + JobStatus m_status; + + /// index within the parent job + int index_within_job = 0; + +signals: + void started(int index); + void progress(int index, qint64 current, qint64 total); + void succeeded(int index); + void failed(int index); + +protected slots: + virtual void downloadProgress(qint64 bytesReceived, qint64 bytesTotal) = 0; + virtual void downloadError(QNetworkReply::NetworkError error) = 0; + virtual void downloadFinished() = 0; + virtual void downloadReadyRead() = 0; + +public slots: + virtual void start() = 0; +}; + +typedef QSharedPointer<Download> DownloadPtr; diff --git a/logic/net/DownloadJob.cpp b/logic/net/DownloadJob.cpp index cad9ae72..cbc2c5fe 100644 --- a/logic/net/DownloadJob.cpp +++ b/logic/net/DownloadJob.cpp @@ -1,136 +1,29 @@ #include "DownloadJob.h" #include "pathutils.h" #include "MultiMC.h" +#include "FileDownload.h" +#include "ByteArrayDownload.h" +#include "CacheDownload.h" -Download::Download (QUrl url, QString target_path, QString expected_md5 ) - :Job() +ByteArrayDownloadPtr DownloadJob::add ( QUrl url ) { - m_url = url; - m_target_path = target_path; - m_expected_md5 = expected_md5; - - m_check_md5 = m_expected_md5.size(); - m_save_to_file = m_target_path.size(); - m_status = Job_NotStarted; - m_opened_for_saving = false; -} - -void Download::start() -{ - if ( m_save_to_file ) - { - QString filename = m_target_path; - m_output_file.setFileName ( filename ); - // if there already is a file and md5 checking is in effect and it can be opened - if ( m_output_file.exists() && m_output_file.open ( QIODevice::ReadOnly ) ) - { - // check the md5 against the expected one - QString hash = QCryptographicHash::hash ( m_output_file.readAll(), QCryptographicHash::Md5 ).toHex().constData(); - m_output_file.close(); - // skip this file if they match - if ( m_check_md5 && hash == m_expected_md5 ) - { - qDebug() << "Skipping " << m_url.toString() << ": md5 match."; - emit succeeded(index_within_job); - return; - } - else - { - m_expected_md5 = hash; - } - } - if(!ensureFilePathExists(filename)) - { - emit failed(index_within_job); - return; - } - } - qDebug() << "Downloading " << m_url.toString(); - QNetworkRequest request ( m_url ); - request.setRawHeader(QString("If-None-Match").toLatin1(), m_expected_md5.toLatin1()); - - auto worker = MMC->qnam(); - QNetworkReply * rep = worker->get ( request ); - - m_reply = QSharedPointer<QNetworkReply> ( rep, &QObject::deleteLater ); - connect ( rep, SIGNAL ( downloadProgress ( qint64,qint64 ) ), SLOT ( downloadProgress ( qint64,qint64 ) ) ); - connect ( rep, SIGNAL ( finished() ), SLOT ( downloadFinished() ) ); - connect ( rep, SIGNAL ( error ( QNetworkReply::NetworkError ) ), SLOT ( downloadError ( QNetworkReply::NetworkError ) ) ); - connect ( rep, SIGNAL ( readyRead() ), SLOT ( downloadReadyRead() ) ); -} - -void Download::downloadProgress ( qint64 bytesReceived, qint64 bytesTotal ) -{ - emit progress (index_within_job, bytesReceived, bytesTotal ); -} - -void Download::downloadError ( QNetworkReply::NetworkError error ) -{ - // error happened during download. - // TODO: log the reason why - m_status = Job_Failed; -} - -void Download::downloadFinished() -{ - // if the download succeeded - if ( m_status != Job_Failed ) - { - // nothing went wrong... - m_status = Job_Finished; - // save the data to the downloadable if we aren't saving to file - if ( !m_save_to_file ) - { - m_data = m_reply->readAll(); - } - else - { - m_output_file.close(); - } - - //TODO: check md5 here! - m_reply.clear(); - emit succeeded(index_within_job); - return; - } - // else the download failed - else - { - if ( m_save_to_file ) - { - m_output_file.close(); - m_output_file.remove(); - } - m_reply.clear(); - emit failed(index_within_job); - return; - } + ByteArrayDownloadPtr ptr (new ByteArrayDownload(url)); + ptr->index_within_job = downloads.size(); + downloads.append(ptr); + return ptr; } -void Download::downloadReadyRead() +FileDownloadPtr DownloadJob::add ( QUrl url, QString rel_target_path) { - if( m_save_to_file ) - { - if(!m_opened_for_saving) - { - if ( !m_output_file.open ( QIODevice::WriteOnly ) ) - { - /* - * Can't open the file... the job failed - */ - m_reply->abort(); - emit failed(index_within_job); - return; - } - m_opened_for_saving = true; - } - m_output_file.write ( m_reply->readAll() ); - } + FileDownloadPtr ptr (new FileDownload(url, rel_target_path)); + ptr->index_within_job = downloads.size(); + downloads.append(ptr); + return ptr; } -DownloadPtr DownloadJob::add ( QUrl url, QString rel_target_path, QString expected_md5 ) +CacheDownloadPtr DownloadJob::add ( QUrl url, MetaEntryPtr entry) { - DownloadPtr ptr (new Download(url, rel_target_path, expected_md5)); + CacheDownloadPtr ptr (new CacheDownload(url, entry)); ptr->index_within_job = downloads.size(); downloads.append(ptr); return ptr; @@ -139,16 +32,17 @@ DownloadPtr DownloadJob::add ( QUrl url, QString rel_target_path, QString expect void DownloadJob::partSucceeded ( int index ) { num_succeeded++; + qDebug() << m_job_name.toLocal8Bit() << " progress: " << num_succeeded << "/" << downloads.size(); if(num_failed + num_succeeded == downloads.size()) { if(num_failed) { - qDebug() << "Download JOB failed: " << this; + qDebug() << m_job_name.toLocal8Bit() << " failed."; emit failed(); } else { - qDebug() << "Download JOB succeeded: " << this; + qDebug() << m_job_name.toLocal8Bit() << " succeeded."; emit succeeded(); } } @@ -159,7 +53,7 @@ void DownloadJob::partFailed ( int index ) num_failed++; if(num_failed + num_succeeded == downloads.size()) { - qDebug() << "Download JOB failed: " << this; + qDebug() << m_job_name.toLocal8Bit() << " failed."; emit failed(); } } @@ -172,7 +66,7 @@ void DownloadJob::partProgress ( int index, qint64 bytesReceived, qint64 bytesTo void DownloadJob::start() { - qDebug() << "Download JOB started: " << this; + qDebug() << m_job_name.toLocal8Bit() << " started."; for(auto iter: downloads) { connect(iter.data(), SIGNAL(succeeded(int)), SLOT(partSucceeded(int))); diff --git a/logic/net/DownloadJob.h b/logic/net/DownloadJob.h index adeef646..71466282 100644 --- a/logic/net/DownloadJob.h +++ b/logic/net/DownloadJob.h @@ -1,98 +1,28 @@ #pragma once #include <QtNetwork> +#include "Download.h" +#include "ByteArrayDownload.h" +#include "FileDownload.h" +#include "CacheDownload.h" +#include "HttpMetaCache.h" class DownloadJob; -class Download; typedef QSharedPointer<DownloadJob> DownloadJobPtr; -typedef QSharedPointer<Download> DownloadPtr; - -enum JobStatus -{ - Job_NotStarted, - Job_InProgress, - Job_Finished, - Job_Failed -}; - -class Job : public QObject -{ - Q_OBJECT -protected: - explicit Job(): QObject(0){}; -public: - virtual ~Job() {}; - -public slots: - virtual void start() = 0; -}; - -class Download: public Job -{ - friend class DownloadJob; - Q_OBJECT -protected: - Download(QUrl url, QString rel_target_path = QString(), QString expected_md5 = QString()); -public: - /// the network reply - QSharedPointer<QNetworkReply> m_reply; - /// source URL - QUrl m_url; - - /// if true, check the md5sum against a provided md5sum - /// also, if a file exists, perform an md5sum first and don't download only if they don't match - bool m_check_md5; - /// the expected md5 checksum - QString m_expected_md5; - - /// save to file? - bool m_save_to_file; - /// is the saving file already open? - bool m_opened_for_saving; - /// if saving to file, use the one specified in this string - QString m_target_path; - /// this is the output file, if any - QFile m_output_file; - /// if not saving to file, downloaded data is placed here - QByteArray m_data; - - int currentProgress = 0; - int totalProgress = 0; - - /// The file's status - JobStatus m_status; - - int index_within_job = 0; -signals: - void started(int index); - void progress(int index, qint64 current, qint64 total); - void succeeded(int index); - void failed(int index); -public slots: - virtual void start(); - -private slots: - void downloadProgress(qint64 bytesReceived, qint64 bytesTotal);; - void downloadError(QNetworkReply::NetworkError error); - void downloadFinished(); - void downloadReadyRead(); -}; /** * A single file for the downloader/cache to process. */ -class DownloadJob : public Job +class DownloadJob : public QObject { Q_OBJECT public: - explicit DownloadJob() - :Job(){}; - explicit DownloadJob(QUrl url, QString rel_target_path = QString(), QString expected_md5 = QString()) - :Job() - { - add(url, rel_target_path, expected_md5); - }; + explicit DownloadJob(QString job_name) + :QObject(), m_job_name(job_name){}; + + ByteArrayDownloadPtr add(QUrl url); + FileDownloadPtr add(QUrl url, QString rel_target_path); + CacheDownloadPtr add(QUrl url, MetaEntryPtr entry); - DownloadPtr add(QUrl url, QString rel_target_path = QString(), QString expected_md5 = QString()); DownloadPtr operator[](int index) { return downloads[index]; @@ -119,6 +49,7 @@ private slots: void partSucceeded(int index); void partFailed(int index); private: + QString m_job_name; QList<DownloadPtr> downloads; int num_succeeded = 0; int num_failed = 0; diff --git a/logic/net/FileDownload.cpp b/logic/net/FileDownload.cpp new file mode 100644 index 00000000..aad4a991 --- /dev/null +++ b/logic/net/FileDownload.cpp @@ -0,0 +1,111 @@ +#include "MultiMC.h" +#include "FileDownload.h" +#include <pathutils.h> +#include <QCryptographicHash> +#include <QDebug> + + +FileDownload::FileDownload ( QUrl url, QString target_path ) + :Download() +{ + m_url = url; + m_target_path = target_path; + m_check_md5 = false; + m_status = Job_NotStarted; + m_opened_for_saving = false; +} + +void FileDownload::start() +{ + QString filename = m_target_path; + m_output_file.setFileName ( filename ); + // if there already is a file and md5 checking is in effect and it can be opened + if ( m_output_file.exists() && m_output_file.open ( QIODevice::ReadOnly ) ) + { + // check the md5 against the expected one + QString hash = QCryptographicHash::hash ( m_output_file.readAll(), QCryptographicHash::Md5 ).toHex().constData(); + m_output_file.close(); + // skip this file if they match + if ( m_check_md5 && hash == m_expected_md5 ) + { + qDebug() << "Skipping " << m_url.toString() << ": md5 match."; + emit succeeded(index_within_job); + return; + } + else + { + m_expected_md5 = hash; + } + } + if(!ensureFilePathExists(filename)) + { + emit failed(index_within_job); + return; + } + + qDebug() << "Downloading " << m_url.toString(); + QNetworkRequest request ( m_url ); + request.setRawHeader(QString("If-None-Match").toLatin1(), m_expected_md5.toLatin1()); + + auto worker = MMC->qnam(); + QNetworkReply * rep = worker->get ( request ); + + m_reply = QSharedPointer<QNetworkReply> ( rep, &QObject::deleteLater ); + connect ( rep, SIGNAL ( downloadProgress ( qint64,qint64 ) ), SLOT ( downloadProgress ( qint64,qint64 ) ) ); + connect ( rep, SIGNAL ( finished() ), SLOT ( downloadFinished() ) ); + connect ( rep, SIGNAL ( error ( QNetworkReply::NetworkError ) ), SLOT ( downloadError ( QNetworkReply::NetworkError ) ) ); + connect ( rep, SIGNAL ( readyRead() ), SLOT ( downloadReadyRead() ) ); +} + +void FileDownload::downloadProgress ( qint64 bytesReceived, qint64 bytesTotal ) +{ + emit progress (index_within_job, bytesReceived, bytesTotal ); +} + +void FileDownload::downloadError ( QNetworkReply::NetworkError error ) +{ + // error happened during download. + // TODO: log the reason why + m_status = Job_Failed; +} + +void FileDownload::downloadFinished() +{ + // if the download succeeded + if ( m_status != Job_Failed ) + { + // nothing went wrong... + m_status = Job_Finished; + m_output_file.close(); + + m_reply.clear(); + emit succeeded(index_within_job); + return; + } + // else the download failed + else + { + m_output_file.close(); + m_reply.clear(); + emit failed(index_within_job); + return; + } +} + +void FileDownload::downloadReadyRead() +{ + if(!m_opened_for_saving) + { + if ( !m_output_file.open ( QIODevice::WriteOnly ) ) + { + /* + * Can't open the file... the job failed + */ + m_reply->abort(); + emit failed(index_within_job); + return; + } + m_opened_for_saving = true; + } + m_output_file.write ( m_reply->readAll() ); +} diff --git a/logic/net/FileDownload.h b/logic/net/FileDownload.h new file mode 100644 |