diff options
Diffstat (limited to 'src/main/kotlin/gui/config/storage')
6 files changed, 603 insertions, 0 deletions
diff --git a/src/main/kotlin/gui/config/storage/ConfigLoadContext.kt b/src/main/kotlin/gui/config/storage/ConfigLoadContext.kt new file mode 100644 index 0000000..4a06ec6 --- /dev/null +++ b/src/main/kotlin/gui/config/storage/ConfigLoadContext.kt @@ -0,0 +1,98 @@ +package moe.nea.firmament.gui.config.storage + +import java.io.PrintWriter +import java.nio.file.Path +import org.apache.commons.io.output.StringBuilderWriter +import kotlin.io.path.ExperimentalPathApi +import kotlin.io.path.OnErrorResult +import kotlin.io.path.Path +import kotlin.io.path.copyToRecursively +import kotlin.io.path.createParentDirectories +import kotlin.io.path.writeText +import moe.nea.firmament.Firmament + +data class ConfigLoadContext( + val loadId: String, +) : AutoCloseable { + val backupPath = Path("backups").resolve(Firmament.MOD_ID) + .resolve("config-$loadId") + .toAbsolutePath() + val logFile = Path("logs") + .resolve(Firmament.MOD_ID) + .resolve("config-$loadId.log") + .toAbsolutePath() + val logBuffer = StringBuilder() + + var shouldSaveLogBuffer = false + fun markShouldSaveLogBuffer() { + shouldSaveLogBuffer = true + } + + fun logDebug(message: String) { + logBuffer.append("[DEBUG] ").append(message).appendLine() + } + + fun logInfo(message: String) { + if (Firmament.DEBUG) + Firmament.logger.info("[ConfigUpgrade] $message") + logBuffer.append("[INFO] ").append(message).appendLine() + } + + fun logError(message: String, exception: Throwable) { + markShouldSaveLogBuffer() + if (Firmament.DEBUG) + Firmament.logger.error("[ConfigUpgrade] $message", exception) + logBuffer.append("[ERROR] ").append(message).appendLine() + PrintWriter(StringBuilderWriter(logBuffer)).use { + exception.printStackTrace(it) + } + logBuffer.appendLine() + } + + fun logError(message: String) { + markShouldSaveLogBuffer() + Firmament.logger.error("[ConfigUpgrade] $message") + logBuffer.append("[ERROR] ").append(message).appendLine() + } + + fun ensureWritable(path: Path) { + path.createParentDirectories() + } + + fun use(block: (ConfigLoadContext) -> Unit) { + try { + block(this) + } catch (ex: Exception) { + logError("Caught exception on CLC", ex) + } finally { + close() + } + } + + override fun close() { + logInfo("Closing out config load.") + if (shouldSaveLogBuffer) { + try { + ensureWritable(logFile) + logFile.writeText(logBuffer.toString()) + } catch (ex: Exception) { + logError("Could not save config load log", ex) + } + } + } + + @OptIn(ExperimentalPathApi::class) + fun createBackup(folder: Path, string: String) { + val backupDestination = backupPath.resolve("$string-${System.currentTimeMillis()}") + logError("Creating backup of $folder in $backupDestination") + folder.copyToRecursively( + backupDestination.createParentDirectories(), + onError = { source: Path, target: Path, exception: Exception -> + logError("Failed to copy subtree $source to $target", exception) + OnErrorResult.SKIP_SUBTREE + }, + followLinks = false, + overwrite = false + ) + } +} diff --git a/src/main/kotlin/gui/config/storage/ConfigStorageClass.kt b/src/main/kotlin/gui/config/storage/ConfigStorageClass.kt new file mode 100644 index 0000000..8258fe7 --- /dev/null +++ b/src/main/kotlin/gui/config/storage/ConfigStorageClass.kt @@ -0,0 +1,8 @@ +package moe.nea.firmament.gui.config.storage + +enum class ConfigStorageClass { // TODO: make this encode type info somehow + PROFILE, + STORAGE, + CONFIG, +} + diff --git a/src/main/kotlin/gui/config/storage/FirmamentConfigLoader.kt b/src/main/kotlin/gui/config/storage/FirmamentConfigLoader.kt new file mode 100644 index 0000000..0292721 --- /dev/null +++ b/src/main/kotlin/gui/config/storage/FirmamentConfigLoader.kt @@ -0,0 +1,252 @@ +package moe.nea.firmament.gui.config.storage + +import java.util.UUID +import java.util.concurrent.CompletableFuture +import kotlinx.serialization.json.JsonElement +import kotlinx.serialization.json.JsonObject +import kotlinx.serialization.json.buildJsonObject +import kotlinx.serialization.json.jsonObject +import kotlin.io.path.Path +import kotlin.io.path.exists +import kotlin.io.path.forEachDirectoryEntry +import kotlin.io.path.isDirectory +import kotlin.io.path.listDirectoryEntries +import kotlin.io.path.name +import kotlin.io.path.readText +import kotlin.io.path.writeText +import kotlin.time.Duration.Companion.seconds +import moe.nea.firmament.annotations.Subscribe +import moe.nea.firmament.events.TickEvent +import moe.nea.firmament.features.debug.DebugLogger +import moe.nea.firmament.util.SBData.NULL_UUID +import moe.nea.firmament.util.TimeMark +import moe.nea.firmament.util.data.IConfigProvider +import moe.nea.firmament.util.data.IDataHolder +import moe.nea.firmament.util.data.ProfileKeyedConfig +import moe.nea.firmament.util.json.intoGson +import moe.nea.firmament.util.json.intoKotlinJson + +object FirmamentConfigLoader { + val currentConfigVersion = 1000 + val configFolder = Path("config/firmament") + .toAbsolutePath() + val storageFolder = configFolder.resolve("storage") + val profilePath = configFolder.resolve("profiles") + val tagLines = listOf( + "<- your config version here", + "I'm a teapot", + "mail.example.com ESMTP", + "Apples" + ) + val configVersionFile = configFolder.resolve("config.version") + + fun loadConfig() { + if (configFolder.exists()) { + if (!configVersionFile.exists()) { + LegacyImporter.importFromLegacy() + } + updateConfigs() + } + + ConfigLoadContext("load-${System.currentTimeMillis()}").use { loadContext -> + val configData = FirstLevelSplitJsonFolder(loadContext, configFolder).load() + loadConfigFromData(configData, Unit, ConfigStorageClass.CONFIG) + val storageData = FirstLevelSplitJsonFolder(loadContext, storageFolder).load() + loadConfigFromData(storageData, Unit, ConfigStorageClass.STORAGE) + var profileData = + profilePath.takeIf { it.exists() } + ?.listDirectoryEntries() + ?.filter { it.isDirectory() } + ?.mapNotNull { + val uuid= runCatching { UUID.fromString(it.name) }.getOrNull() ?: return@mapNotNull null + uuid to FirstLevelSplitJsonFolder(loadContext, it).load() + } + ?.toMap() + if (profileData.isNullOrEmpty()) + profileData = mapOf(NULL_UUID to JsonObject(mapOf())) + profileData.forEach { (key, value) -> + loadConfigFromData(value, key, ConfigStorageClass.PROFILE) + } + } + } + + fun <T> loadConfigFromData( + configData: JsonObject, + key: T?, + storageClass: ConfigStorageClass + ) { + for (holder in allConfigs) { + if (holder.storageClass == storageClass) { + val h = (holder as IDataHolder<T>) + if (key == null) { + h.explicitDefaultLoad() + } else { + h.loadFrom(key, configData) + } + } + } + } + + fun <T> collectConfigFromData( + key: T, + storageClass: ConfigStorageClass, + ): JsonObject { + var json = JsonObject(mapOf()) + for (holder in allConfigs) { + if (holder.storageClass == storageClass) { + json = mergeJson(json, (holder as IDataHolder<T>).saveTo(key)) + } + } + return json + } + + fun <T> saveStorage( + storageClass: ConfigStorageClass, + key: T, + firstLevelSplitJsonFolder: FirstLevelSplitJsonFolder, + ) { + firstLevelSplitJsonFolder.save( + collectConfigFromData(key, storageClass) + ) + } + + fun collectAllProfileIds(): Set<UUID> { + return allConfigs + .filter { it.storageClass == ConfigStorageClass.PROFILE } + .flatMapTo(mutableSetOf()) { + (it as ProfileKeyedConfig<*>).keys() + } + } + + fun saveAll() { + ConfigLoadContext("save-${System.currentTimeMillis()}").use { context -> + saveStorage( + ConfigStorageClass.CONFIG, + Unit, + FirstLevelSplitJsonFolder(context, configFolder) + ) + saveStorage( + ConfigStorageClass.STORAGE, + Unit, + FirstLevelSplitJsonFolder(context, storageFolder) + ) + collectAllProfileIds().forEach { profileId -> + saveStorage( + ConfigStorageClass.PROFILE, + profileId, + FirstLevelSplitJsonFolder(context, profilePath.resolve(profileId.toString())) + ) + } + writeConfigVersion() + } + } + + fun mergeJson(a: JsonObject, b: JsonObject): JsonObject { + fun mergeInner(a: JsonElement?, b: JsonElement?): JsonElement { + if (a == null) + return b!! + if (b == null) + return a + a as JsonObject + b as JsonObject + return buildJsonObject { + (a.keys + b.keys) + .forEach { + put(it, mergeInner(a[it], b[it])) + } + } + } + return mergeInner(a, b) as JsonObject + } + + val allConfigs: List<IDataHolder<*>> = IConfigProvider.providers.allValidInstances.flatMap { it.configs } + + fun updateConfigs() { + val startVersion = configVersionFile.readText() + .substringBefore(' ') + .trim() + .toInt() + ConfigLoadContext("update-from-$startVersion-to-$currentConfigVersion-${System.currentTimeMillis()}") + .use { loadContext -> + updateOneConfig( + loadContext, + startVersion, + ConfigStorageClass.CONFIG, + FirstLevelSplitJsonFolder(loadContext, configFolder) + ) + updateOneConfig( + loadContext, + startVersion, + ConfigStorageClass.STORAGE, + FirstLevelSplitJsonFolder(loadContext, storageFolder) + ) + profilePath.forEachDirectoryEntry { + updateOneConfig( + loadContext, + startVersion, + ConfigStorageClass.PROFILE, + FirstLevelSplitJsonFolder(loadContext, it) + ) + } + writeConfigVersion() + } + } + + fun writeConfigVersion() { + configVersionFile.writeText("$currentConfigVersion ${tagLines.random()}") + } + + private fun updateOneConfig( + loadContext: ConfigLoadContext, + startVersion: Int, + storageClass: ConfigStorageClass, + firstLevelSplitJsonFolder: FirstLevelSplitJsonFolder + ) { + if (startVersion == currentConfigVersion) { + loadContext.logDebug("Skipping upgrade to ") + return + } + loadContext.logInfo("Starting upgrade from at ${firstLevelSplitJsonFolder.folder} ($storageClass) to $startVersion") + var data = firstLevelSplitJsonFolder.load() + for (nextVersion in (startVersion + 1)..currentConfigVersion) { + data = updateOneConfigOnce(nextVersion, storageClass, data) + } + firstLevelSplitJsonFolder.save(data) + } + + private fun updateOneConfigOnce( + nextVersion: Int, + storageClass: ConfigStorageClass, + data: JsonObject + ): JsonObject { + return ConfigFixEvent.publish(ConfigFixEvent(storageClass, nextVersion, data.intoGson().asJsonObject)) + .data.intoKotlinJson().jsonObject + } + + @Subscribe + fun onTick(event: TickEvent) { + val config = configPromise ?: return + val passedTime = saveDebounceStart.passedTime() + if (passedTime < 1.seconds) + return + if (!config.isDone && passedTime < 3.seconds) + return + debugLogger.log("Performing config save") + configPromise = null + saveAll() + } + + val debugLogger = DebugLogger("config") + + var configPromise: CompletableFuture<Void?>? = null + var saveDebounceStart: TimeMark = TimeMark.farPast() + fun markDirty( + holder: IDataHolder<*>, + timeoutPromise: CompletableFuture<Void?>? = null + ) { + debugLogger.log("Config marked dirty") + this.saveDebounceStart = TimeMark.now() + this.configPromise = timeoutPromise ?: CompletableFuture.completedFuture(null) + } + +} diff --git a/src/main/kotlin/gui/config/storage/FirstLevelSplitJsonFolder.kt b/src/main/kotlin/gui/config/storage/FirstLevelSplitJsonFolder.kt new file mode 100644 index 0000000..b92488a --- /dev/null +++ b/src/main/kotlin/gui/config/storage/FirstLevelSplitJsonFolder.kt @@ -0,0 +1,109 @@ +@file:OptIn(ExperimentalSerializationApi::class) + +package moe.nea.firmament.gui.config.storage + +import java.nio.file.Path +import kotlinx.serialization.ExperimentalSerializationApi +import kotlinx.serialization.json.JsonElement +import kotlinx.serialization.json.JsonObject +import kotlinx.serialization.json.decodeFromStream +import kotlinx.serialization.json.encodeToStream +import kotlin.io.path.createDirectories +import kotlin.io.path.deleteExisting +import kotlin.io.path.exists +import kotlin.io.path.inputStream +import kotlin.io.path.listDirectoryEntries +import kotlin.io.path.nameWithoutExtension +import kotlin.io.path.outputStream +import moe.nea.firmament.Firmament + +// TODO: make this class write / read async +class FirstLevelSplitJsonFolder( + val context: ConfigLoadContext, + val folder: Path +) { + + var hasCreatedBackup = false + + fun backup(cause: String) { + if (hasCreatedBackup) return + hasCreatedBackup = true + context.createBackup(folder, cause) + } + + fun load(): JsonObject { + context.logInfo("Loading FLSJF from $folder") + if (!folder.exists()) + return JsonObject(mapOf()) + return try { + folder.listDirectoryEntries("*.json") + .mapNotNull(::loadIndividualFile) + .toMap() + .let(::JsonObject) + .also { context.logInfo("FLSJF from $folder - Voller Erfolg!") } + } catch (ex: Exception) { + context.logError("Could not load files from $folder", ex) + backup("failed-load") + JsonObject(mapOf()) + } + } + + fun loadIndividualFile(path: Path): Pair<String, JsonElement>? { + context.logDebug("Loading partial file from $path") + return try { + path.inputStream().use { + path.nameWithoutExtension to Firmament.json.decodeFromStream(JsonElement.serializer(), it) + } + } catch (ex: Exception) { + context.logError("Could not load file from $path", ex) + backup("failed-load") + null + } + } + + fun save(value: JsonObject) { + context.logInfo("Saving FLSJF to $folder") + context.logDebug("Current value:\n$value") + if (!folder.exists()) { + context.logInfo("Creating folder $folder") + folder.createDirectories() + } + val entries = folder.listDirectoryEntries("*.json") + .toMutableList() + for ((name, element) in value) { + val path = saveIndividualFile(name, element) + if (path != null) { + entries.remove(path) + } + } + if (entries.isNotEmpty()) { + context.logInfo("Deleting additional files.") + for (path in entries) { + context.logInfo("Deleting $path") + backup("save-deletion") + try { + path.deleteExisting() + } catch (ex: Exception) { + context.logError("Could not delete $path", ex) + } + } + } + context.logInfo("FLSJF to $folder - Voller Erfolg!") + } + + fun saveIndividualFile(name: String, element: JsonElement): Path? { + try { + context.logDebug("Saving partial file with name $name") + val path = folder.resolve("$name.json") + context.ensureWritable(path) + path.outputStream().use { + Firmament.json.encodeToStream(JsonElement.serializer(), element, it) + } + return path + } catch (ex: Exception) { + context.logError("Could not save $name with value $element", ex) + backup("failed-save") + return null + } + } +} diff --git a/src/main/kotlin/gui/config/storage/LegacyImporter.kt b/src/main/kotlin/gui/config/storage/LegacyImporter.kt new file mode 100644 index 0000000..c1f8b90 --- /dev/null +++ b/src/main/kotlin/gui/config/storage/LegacyImporter.kt @@ -0,0 +1,68 @@ +package moe.nea.firmament.gui.config.storage + +import java.nio.file.Path +import kotlin.io.path.copyTo +import kotlin.io.path.createDirectories +import kotlin.io.path.createParentDirectories +import kotlin.io.path.exists +import kotlin.io.path.forEachDirectoryEntry +import kotlin.io.path.listDirectoryEntries +import kotlin.io.path.moveTo +import kotlin.io.path.name +import kotlin.io.path.nameWithoutExtension +import kotlin.io.path.writeText +import moe.nea.firmament.gui.config.storage.FirmamentConfigLoader.configFolder +import moe.nea.firmament.gui.config.storage.FirmamentConfigLoader.configVersionFile +import moe.nea.firmament.gui.config.storage.FirmamentConfigLoader.storageFolder + +object LegacyImporter { + val legacyConfigVersion = 995 + val backupPath = configFolder.resolveSibling("firmament-legacy-config-${System.currentTimeMillis()}") + + fun copyIf(from: Path, to: Path) { + if (from.exists()) { + to.createParentDirectories() + from.copyTo(to) + } + } + + val legacyStorage = listOf( + "inventory-buttons", + "macros", + ) + + fun importFromLegacy() { + if (!configFolder.exists()) return + configFolder.moveTo(backupPath) + configFolder.createDirectories() + + legacyStorage.forEach { + copyIf( + backupPath.resolve("$it.json"), + storageFolder.resolve("$it.json") + ) + } + + backupPath.listDirectoryEntries("*.json") + .filter { it.nameWithoutExtension !in legacyStorage } + .forEach { path -> + val name = path.name + path.copyTo(configFolder.resolve(name)) + } + + backupPath.resolve("profiles") + .takeIf { it.exists() } + ?.forEachDirectoryEntry { category -> + category.forEachDirectoryEntry { profile -> + copyIf( + profile, + FirmamentConfigLoader.profilePath + .resolve(profile.nameWithoutExtension) + .resolve(category.name + ".json") + ) + } + } + + configVersionFile.writeText("$legacyConfigVersion LEGACY") + } +} diff --git a/src/main/kotlin/gui/config/storage/README.md b/src/main/kotlin/gui/config/storage/README.md new file mode 100644 index 0000000..aad4afe --- /dev/null +++ b/src/main/kotlin/gui/config/storage/README.md @@ -0,0 +1,68 @@ +<!-- +SPDX-FileCopyrightText: 2025 Linnea Gräf <nea@nea.moe> + +SPDX-License-Identifier: CC0-1.0 +--> + +# Plan for the 2026 Config Renewal of Firmament + +The current config system in Firmament is not growing at a reasonable pace. Here is a list of my grievances with it: + +- the config files are split, resulting in making migrations between different config files (which might load in + different order) difficult +- it is difficult to detect extraneous properties / files, because not all files are loaded and consumed at once +- profile specific data should be in a different hierarchy. the current hierarchy of `profiles/topic/<uuid>.json` orders + data from different profiles to be closer than data from the same profile. this also contributes to the two former + problems. + +## Goals + +- i want to retain having multiple different files for different topics, as well as a folder structure that makes sense + for profiles. +- i want to split up "storage" type data, with "config" type data +- i want to support partial loads with some broken files (resetting the files that are broken) +- i want to support backups on any detected error (or simply at will) + - notably i do not care about the structure of the backups much. even just a all json files merged backup is fine + for me, for now. + +## Implementation + +### FirstLevelSplitJsonFolder + +One of the basic components of this new config folder is a `FirstLevelSplitJsonFolder`. A `FLSJF` takes in a folder +containing multiple JSON-files and loads all of them unconditionally. Each file is then inserted side by side into a +json object, to be processed further by other mechanisms. + +In essence the `FLSJF` takes a folder structure like this: + +``` +file-1.json +file-2.json +file-3.json +``` + +and turns it into a single merged json object: + +```json +{ + "file-1": "the json content of file-1.json", + "file-2": "the json content of file-2.json", + "file-3": "the json content of file-3.json" +} +``` + +As with any stage of the implementation, any unparsable files shall be copied over to a backup spot and discarded. + +Nota bene: Folders are wholesale ignored. + +### Config folders + +Firmament stores all configs and data in the root config folder `./config/firmament`. + +- Any config data is stored as an [`FLSJF`](#firstlevelsplitjsonfolder) in the root config folder +- Any generic storage data is stored as an [`FLSJF`](#firstlevelsplitjsonfolder) in `${rootConfigFolder}/storage/`. +- Any profile specific storage data is stored as an [`FLSJF`](#firstlevelsplitjsonfolder) for each profile in `${rootConfigFolder}/profileStorage/${profileUuid}/`. +- Any backup data is stored in `${rootConfigFolder}/backups/${launchId}/${loadId}/${fileName}`. + - Where `launchId` is `${currentLaunchTimestamp}-${random()}` to avoid collisions. + - Where `loadId` depends on which stage of the config load we are doing (`merge`/`upgrade`/etc.) and what type of config we are loading (`profileSpecific`/`config`/etc.). + - And where `fileName` may be a relative filename of where this data was originally found or some internal descriptor for the merged data stage we are on. |