repo downloading

8 files changed, 223 insertions, 29 deletions

diff --git a/build.gradle.kts b/build.gradle.kts
index 548f211..c439e86 100644
--- a/build.gradle.kts
+++ b/build.gradle.kts
@@ -4,17 +4,21 @@ plugins {
     java
     `maven-publish`
     kotlin("jvm") version "1.7.10"
+    kotlin("plugin.serialization") version "1.7.10"
     id("dev.architectury.loom") version "0.12.0.+"
     id("com.github.johnrengelman.shadow") version "7.1.2"
 }
 
 loom {
     accessWidenerPath.set(project.file("src/main/resources/notenoughupdates.accesswidener"))
-    launches {
+    runConfigs {
         removeIf { it.name != "client" }
+    }
+    launches {
         named("client") {
             property("devauth.enabled", "true")
             property("fabric.log.level", "info")
+            property("notenoughupdates.debug", "true")
         }
     }
 }
@@ -35,6 +39,13 @@ val shadowMe by configurations.creating {
     configurations.implementation.get().extendsFrom(this)
 }
 
+val transInclude by configurations.creating {
+    exclude(group = "com.mojang")
+    exclude(group = "org.jetbrains.kotlin")
+    exclude(group = "org.jetbrains.kotlinx")
+    isTransitive = true
+}
+
 dependencies {
     // Minecraft dependencies
     "minecraft"("com.mojang:minecraft:${project.property("minecraft_version")}")
@@ -48,12 +59,20 @@ dependencies {
     // Actual dependencies
     modCompileOnly("me.shedaniel:RoughlyEnoughItems-api:${rootProject.property("rei_version")}")
     shadowMe("io.github.moulberry:neurepoparser:0.0.1")
+    fun ktor(mod: String) = "io.ktor:ktor-$mod-jvm:${project.property("ktor_version")}"
+    transInclude(implementation(ktor("client-core"))!!)
+    transInclude(implementation(ktor("client-java"))!!)
+    transInclude(implementation(ktor("serialization-kotlinx-json"))!!)
+    transInclude(implementation(ktor("client-content-negotiation"))!!)
 
     // Dev environment preinstalled mods
     modRuntimeOnly("me.shedaniel:RoughlyEnoughItems-fabric:${project.property("rei_version")}")
     modRuntimeOnly("me.djtheredstoner:DevAuth-fabric:${project.property("devauth_version")}")
     modRuntimeOnly("maven.modrinth:modmenu:${project.property("modmenu_version")}")
 
+    transInclude.resolvedConfiguration.resolvedArtifacts.forEach {
+        include(it.moduleVersion.id.toString())
+    }
 }
 
 
diff --git a/gradle.properties b/gradle.properties
index c9b35eb..1624ae7 100644
--- a/gradle.properties
+++ b/gradle.properties
@@ -16,3 +16,4 @@ fabric_kotlin_version=1.8.2+kotlin.1.7.10
 rei_version=9.1.518
 devauth_version=1.0.0
 modmenu_version=4.0.4
+ktor_version=2.0.3
diff --git a/src/main/kotlin/moe/nea/notenoughupdates/NotEnoughUpdates.kt b/src/main/kotlin/moe/nea/notenoughupdates/NotEnoughUpdates.kt
index 27c51e1..c4f3f69 100644
--- a/src/main/kotlin/moe/nea/notenoughupdates/NotEnoughUpdates.kt
+++ b/src/main/kotlin/moe/nea/notenoughupdates/NotEnoughUpdates.kt
@@ -1,44 +1,71 @@
 package moe.nea.notenoughupdates
 
+import com.mojang.brigadier.Command
 import com.mojang.brigadier.CommandDispatcher
-import io.github.moulberry.repo.NEURepository
-import moe.nea.notenoughupdates.repo.ItemCache
+import io.ktor.client.*
+import io.ktor.client.plugins.*
+import io.ktor.client.plugins.contentnegotiation.*
+import io.ktor.serialization.kotlinx.json.*
+import kotlinx.coroutines.*
+import kotlinx.serialization.json.Json
+import moe.nea.notenoughupdates.repo.RepoManager
 import net.fabricmc.api.ClientModInitializer
 import net.fabricmc.api.ModInitializer
 import net.fabricmc.fabric.api.client.command.v2.ClientCommandManager
 import net.fabricmc.fabric.api.client.command.v2.ClientCommandRegistrationCallback
 import net.fabricmc.fabric.api.client.command.v2.FabricClientCommandSource
-import net.minecraft.client.Minecraft
+import net.fabricmc.loader.api.FabricLoader
 import net.minecraft.commands.CommandBuildContext
 import net.minecraft.network.chat.Component
-import net.minecraft.network.protocol.game.ClientboundUpdateRecipesPacket
+import org.apache.logging.log4j.LogManager
+import java.nio.file.Files
 import java.nio.file.Path
+import kotlin.coroutines.EmptyCoroutineContext
 
 object NotEnoughUpdates : ModInitializer, ClientModInitializer {
-    val DATA_DIR = Path.of(".notenoughupdates")
-
     const val MOD_ID = "notenoughupdates"
 
-    val neuRepo: NEURepository = NEURepository.of(Path.of("NotEnoughUpdates-REPO")).apply {
-        registerReloadListener(ItemCache)
-        reload()
-        registerReloadListener {
-            Minecraft.getInstance().connection?.handleUpdateRecipes(ClientboundUpdateRecipesPacket(mutableListOf()))
+    val DATA_DIR = Path.of(".notenoughupdates").also { Files.createDirectories(it) }
+    val DEBUG = System.getenv("notenoughupdates.debug") == "true"
+    val logger = LogManager.getLogger("NotEnoughUpdates")
+    val metadata by lazy { FabricLoader.getInstance().getModContainer(MOD_ID).orElseThrow().metadata }
+    val version by lazy { metadata.version }
+
+    val json = Json {
+        prettyPrint = DEBUG
+        ignoreUnknownKeys = true
+    }
+
+    val httpClient by lazy {
+        HttpClient {
+            install(ContentNegotiation) {
+                json(json)
+            }
+            install(UserAgent) {
+                agent = "NotEnoughUpdates1.19/$version"
+            }
         }
     }
 
-    fun registerCommands(
-        dispatcher: CommandDispatcher<FabricClientCommandSource>, registryAccess: CommandBuildContext
+    val globalJob = Job()
+    val coroutineScope =
+        CoroutineScope(EmptyCoroutineContext + CoroutineName("NotEnoughUpdates")) + SupervisorJob(globalJob)
+    val coroutineScopeIo = coroutineScope + Dispatchers.IO + SupervisorJob(globalJob)
+
+    private fun registerCommands(
+        dispatcher: CommandDispatcher<FabricClientCommandSource>,
+        @Suppress("UNUSED_PARAMETER")
+        _ctx: CommandBuildContext
     ) {
         dispatcher.register(ClientCommandManager.literal("neureload").executes {
             it.source.sendFeedback(Component.literal("Reloading repository from disk. This may lag a bit."))
-            neuRepo.reload()
-            0
+            RepoManager.neuRepo.reload()
+            Command.SINGLE_SUCCESS
         })
-
     }
 
     override fun onInitialize() {
+        RepoManager.launchAsyncUpdate()
         ClientCommandRegistrationCallback.EVENT.register(this::registerCommands)
     }
 
diff --git a/src/main/kotlin/moe/nea/notenoughupdates/rei/NEUReiPlugin.kt b/src/main/kotlin/moe/nea/notenoughupdates/rei/NEUReiPlugin.kt
index 69cca41..985743a 100644
--- a/src/main/kotlin/moe/nea/notenoughupdates/rei/NEUReiPlugin.kt
+++ b/src/main/kotlin/moe/nea/notenoughupdates/rei/NEUReiPlugin.kt
@@ -6,8 +6,8 @@ import me.shedaniel.rei.api.client.registry.entry.EntryRegistry
 import me.shedaniel.rei.api.common.entry.EntryStack
 import me.shedaniel.rei.api.common.entry.type.EntryTypeRegistry
 import me.shedaniel.rei.api.common.entry.type.VanillaEntryTypes
-import moe.nea.notenoughupdates.NotEnoughUpdates.neuRepo
 import moe.nea.notenoughupdates.repo.ItemCache.asItemStack
+import moe.nea.notenoughupdates.repo.RepoManager
 import net.minecraft.resources.ResourceLocation
 import net.minecraft.world.item.ItemStack
 
@@ -22,13 +22,14 @@ class NEUReiPlugin : REIClientPlugin {
 
         val SKYBLOCK_ITEM_TYPE_ID = ResourceLocation("notenoughupdates", "skyblockitems")
     }
+
     override fun registerEntryTypes(registry: EntryTypeRegistry) {
         registry.register(SKYBLOCK_ITEM_TYPE_ID, SBItemEntryDefinition)
     }
 
 
     override fun registerEntries(registry: EntryRegistry) {
-        neuRepo.items.items.values.forEach {
+        RepoManager.neuRepo.items.items.values.forEach {
             if (!it.isVanilla)
                 registry.addEntry(EntryStack.of(SBItemEntryDefinition, it))
         }
diff --git a/src/main/kotlin/moe/nea/notenoughupdates/repo/ItemCache.kt b/src/main/kotlin/moe/nea/notenoughupdates/repo/ItemCache.kt
index aa93fec..893b1c0 100644
--- a/src/main/kotlin/moe/nea/notenoughupdates/repo/ItemCache.kt
+++ b/src/main/kotlin/moe/nea/notenoughupdates/repo/ItemCache.kt
@@ -4,6 +4,7 @@ import com.mojang.serialization.Dynamic
 import io.github.moulberry.repo.IReloadable
 import io.github.moulberry.repo.NEURepository
 import io.github.moulberry.repo.data.NEUItem
+import moe.nea.notenoughupdates.NotEnoughUpdates
 import moe.nea.notenoughupdates.util.LegacyTagParser
 import moe.nea.notenoughupdates.util.appendLore
 import net.minecraft.nbt.CompoundTag
@@ -37,7 +38,7 @@ object ItemCache : IReloadable {
                 2975
             ).value as CompoundTag
         } catch (e: Exception) {
-            e.printStackTrace()
+            NotEnoughUpdates.logger.error("Failed to datafixer an item", e)
             isFlawless = false
             null
         }
@@ -46,7 +47,7 @@ object ItemCache : IReloadable {
         val oldItemTag = get10809CompoundTag()
         val modernItemTag = oldItemTag.transformFrom10809ToModern()
             ?: return ItemStack(Items.PAINTING).apply {
-                setHoverName(Component.literal(this@asItemStackNow.displayName))
+                hoverName = Component.literal(this@asItemStackNow.displayName)
                 appendLore(listOf(Component.literal("Exception rendering item: $skyblockItemId")))
             }
         val itemInstance = ItemStack.of(modernItemTag)
diff --git a/src/main/kotlin/moe/nea/notenoughupdates/repo/RepoDownloadManager.kt b/src/main/kotlin/moe/nea/notenoughupdates/repo/RepoDownloadManager.kt
index 47b2878..977c035 100644
--- a/src/main/kotlin/moe/nea/notenoughupdates/repo/RepoDownloadManager.kt
+++ b/src/main/kotlin/moe/nea/notenoughupdates/repo/RepoDownloadManager.kt
@@ -1,17 +1,122 @@
 package moe.nea.notenoughupdates.repo
 
+import io.ktor.client.call.*
+import io.ktor.client.request.*
+import io.ktor.client.statement.*
+import io.ktor.utils.io.jvm.nio.*
+import kotlinx.coroutines.CoroutineName
+import kotlinx.coroutines.Dispatchers.IO
+import kotlinx.coroutines.withContext
+import kotlinx.serialization.Serializable
 import moe.nea.notenoughupdates.NotEnoughUpdates
+import moe.nea.notenoughupdates.NotEnoughUpdates.logger
+import moe.nea.notenoughupdates.util.iterate
+import java.io.IOException
+import java.nio.file.Files
+import java.nio.file.Path
+import java.nio.file.StandardOpenOption
+import java.util.zip.ZipInputStream
+import kotlin.io.path.*
+
 
 object RepoDownloadManager {
 
     val repoSavedLocation = NotEnoughUpdates.DATA_DIR.resolve("repo-extracted")
-    val repoMetadataLocation = NotEnoughUpdates.DATA_DIR.resolve("loaded-repo.json")
-
-    data class RepoMetadata(
-        var latestCommit: String,
-        var user: String,
-        var repository: String,
-        var branch: String,
-    )
+    val repoMetadataLocation = NotEnoughUpdates.DATA_DIR.resolve("loaded-repo-sha.txt")
+
+    val user = "NotEnoughUpdates"
+    val repo = "NotEnoughUpdates-REPO"
+    val branch = "dangerous"
+
+    private fun loadSavedVersionHash(): String? =
+        if (repoSavedLocation.exists()) {
+            if (repoMetadataLocation.exists()) {
+                try {
+                    repoMetadataLocation.readText().trim()
+                } catch (e: IOException) {
+                    null
+                }
+            } else {
+                null
+            }
+        } else null
+
+    private fun saveVersionHash(versionHash: String) {
+        latestSavedVersionHash = versionHash
+        repoMetadataLocation.writeText(versionHash)
+    }
+
+    var latestSavedVersionHash: String? = loadSavedVersionHash()
+        private set
+
+    @Serializable
+    private class GithubCommitsResponse(val sha: String)
+
+    private suspend fun requestLatestGithubSha(): String? {
+        val response =
+            NotEnoughUpdates.httpClient.get("https://api.github.com/repos/$user/$repo/commits/$branch")
+        if (response.status.value != 200) {
+            return null
+        }
+        return response.body<GithubCommitsResponse>().sha
+    }
+
+    private suspend fun downloadGithubArchive(url: String): Path = withContext(IO) {
+        val response = NotEnoughUpdates.httpClient.get(url)
+        val targetFile = Files.createTempFile("notenoughupdates-repo", ".zip")
+        val outputChannel = Files.newByteChannel(targetFile, StandardOpenOption.CREATE, StandardOpenOption.WRITE)
+        response.bodyAsChannel().copyTo(outputChannel)
+        targetFile
+    }
+
+    /**
+     * Downloads the latest repository from github, setting [latestSavedVersionHash].
+     * @return true, if an update was performed, false, otherwise (no update needed, or wasn't able to complete update)
+     */
+    suspend fun downloadUpdate(): Boolean = withContext(CoroutineName("Repo Update Check")) {
+        val latestSha = requestLatestGithubSha()
+        if (latestSha == null) {
+            logger.warn("Could not request github API to retrieve latest REPO sha.")
+            return@withContext false
+        }
+        val currentSha = loadSavedVersionHash()
+        if (latestSha != currentSha) {
+            val requestUrl = "https://github.com/$user/$repo/archive/$latestSha.zip"
+            logger.info("Planning to upgrade repository from $currentSha to $latestSha from $requestUrl")
+            val zipFile = downloadGithubArchive(requestUrl)
+            logger.info("Download repository zip file to $zipFile. Deleting old repository")
+            withContext(IO) { repoSavedLocation.deleteIfExists() }
+            logger.info("Extracting new repository")
+            withContext(IO) { extractNewRepository(zipFile) }
+            logger.info("Repository loaded on disk.")
+            saveVersionHash(latestSha)
+            return@withContext true
+        } else {
+            logger.debug("Repository on latest sha $currentSha. Not performing update")
+            return@withContext false
+        }
+    }
+
+    private fun extractNewRepository(zipFile: Path) {
+        repoSavedLocation.createDirectories()
+        ZipInputStream(zipFile.inputStream()).use { cis ->
+            while (true) {
+                val entry = cis.nextEntry ?: break
+                if (entry.isDirectory) continue
+                val extractedLocation =
+                    repoSavedLocation.resolve(
+                        entry.name.substringAfter('/', missingDelimiterValue = "")
+                    )
+                if (repoSavedLocation !in extractedLocation.iterate { it.parent }) {
+                    logger.error("Not Enough Updates detected an invalid zip file. This is a potential security risk, please report this in the Moulberry discord.")
+                    throw RuntimeException("Not Enough Updates detected an invalid zip file. This is a potential security risk, please report this in the Moulberry discord.")
+                }
+                extractedLocation.parent.createDirectories()
+                cis.copyTo(extractedLocation.outputStream())
+                cis.closeEntry()
+            }
+        }
+    }
+
 
 }
diff --git a/src/main/kotlin/moe/nea/notenoughupdates/repo/RepoManager.kt b/src/main/kotlin/moe/nea/notenoughupdates/repo/RepoManager.kt
new file mode 100644
index 0000000..f3d53e8
--- /dev/null
+++ b/src/main/kotlin/moe/nea/notenoughupdates/repo/RepoManager.kt
@@ -0,0 +1,31 @@
+package moe.nea.notenoughupdates.repo
+
+import io.github.moulberry.repo.NEURepository
+import kotlinx.coroutines.launch
+import moe.nea.notenoughupdates.NotEnoughUpdates
+import moe.nea.notenoughupdates.NotEnoughUpdates.logger
+import net.minecraft.client.Minecraft
+import net.minecraft.network.protocol.game.ClientboundUpdateRecipesPacket
+
+object RepoManager {
+
+
+    val neuRepo: NEURepository = NEURepository.of(RepoDownloadManager.repoSavedLocation).apply {
+        registerReloadListener(ItemCache)
+        registerReloadListener {
+            if (Minecraft.getInstance().connection?.handleUpdateRecipes(ClientboundUpdateRecipesPacket(mutableListOf())) == null) {
+                logger.warn("Failed to issue a ClientboundUpdateRecipesPacket (to reload REI). This may lead to an outdated item list.")
+            }
+        }
+    }
+
+
+    fun launchAsyncUpdate() {
+        NotEnoughUpdates.coroutineScope.launch {
+            if (RepoDownloadManager.downloadUpdate()) {
+                neuRepo.reload()
+            }
+        }
+    }
+
+}
diff --git a/src/main/kotlin/moe/nea/notenoughupdates/util/SequenceUtil.kt b/src/main/kotlin/moe/nea/notenoughupdates/util/SequenceUtil.kt
new file mode 100644
index 0000000..3e338bd
--- /dev/null
+++ b/src/main/kotlin/moe/nea/notenoughupdates/util/SequenceUtil.kt
@@ -0,0 +1,9 @@
+package moe.nea.notenoughupdates.util
+
+fun <T : Any> T.iterate(iterator: (T) -> T?): Sequence<T> = sequence {
+    var x: T? = this@iterate
+    while (x != null) {
+        yield(x)
+        x = iterator(x)
+    }
+}