import io.ktor.client.* import io.ktor.client.engine.cio.* import io.ktor.client.request.* import io.ktor.client.statement.* import kotlinx.serialization.* import kotlinx.serialization.descriptors.PolymorphicKind import kotlinx.serialization.descriptors.SerialDescriptor import kotlinx.serialization.descriptors.buildSerialDescriptor import kotlinx.serialization.encoding.Decoder import kotlinx.serialization.json.* import java.io.File @Serializable data class NpcListData( val displayName: String, val id: NpcId, val tag: String? = null, ) @Serializable data class ListData( val data: List ) val json = Json { ignoreUnknownKeys = true } val ListDataPattern = "new Listview\\((.*)\\);\n".toRegex() val NpcLocationDataPattern = "g_mapperData = (.*);\n".toRegex() val client = HttpClient(CIO) suspend fun getNpcList(zoneId: Int): List { return getNpcList("https://www.wowhead.com/npcs/react-a:1?filter=6;$zoneId;0") + getNpcList("https://www.wowhead.com/npcs/react-a:0?filter=6;$zoneId;0") + getNpcList("https://www.wowhead.com/npcs/react-a:-1?filter=6;$zoneId;0") } suspend fun getNpcList(url: String): List { val string = client.get(url).bodyAsText() val match = ListDataPattern.find(string)!! val jsonData = match.groupValues[1] val data = json.decodeFromString>(jsonData).data if (data.size == 1000) { println("Warning: Encountered NPC limit for url $url") } return data } @Serializable @JvmInline value class ZoneId(val int: Int) @Serializable @JvmInline value class MapId(val int: Int) @Serializable @JvmInline value class NpcId(val int: Int) @Serializable data class MapCoordinates( val count: Int, val coords: List>, val uiMapId: MapId? = null, val uiMapName: String? = null, ) @OptIn(InternalSerializationApi::class, ExperimentalSerializationApi::class) object NpcZoneMapSerializer : DeserializationStrategy>> { override val descriptor: SerialDescriptor get() = buildSerialDescriptor("NpcZoneMapSerializer", PolymorphicKind.SEALED) override fun deserialize(decoder: Decoder): Map> { val d = decoder as JsonDecoder val el = d.decodeJsonElement() el as JsonObject return el.mapNotNull { (t, u) -> if (u is JsonArray) ZoneId(t.toInt()) to d.json.decodeFromJsonElement>(u) else null }.toMap() } } suspend fun getNpcData(id: NpcId, name: String): Map> { val string = client.get("https://www.wowhead.com/npc=${id.int}/${name.replace(" ", "-").lowercase()}").bodyAsText() val match = NpcLocationDataPattern.find(string)!! val jsonData = match.groupValues[1] try { return json.decodeFromString(NpcZoneMapSerializer, jsonData) } finally { File("crash.html").writeText(string) } } @Serializable data class CompleteNpcData( val metadata: NpcListData, val locations: Map>, ) @Serializable data class AllNpcData( val npcData: Map ) suspend fun main() { val zoneIdsToScrape = listOf(13646) val npcsPerZone = zoneIdsToScrape.associateWith { getNpcList(it) } val indexedNpcIds = npcsPerZone.flatMap { it.value }.associateBy { it.id } val npcData = indexedNpcIds.values.associate { it.id to CompleteNpcData(it, getNpcData(it.id, it.displayName)) } File("database.json").writeText(json.encodeToString(npcData)) }