diff options
Diffstat (limited to 'build')
| -rw-r--r-- | build/cleaners/rank.js | 10 | ||||
| -rw-r--r-- | build/cleaners/skyblock/profile.js | 7 | ||||
| -rw-r--r-- | build/constants.js | 42 | ||||
| -rw-r--r-- | build/database.js | 64 | ||||
| -rw-r--r-- | build/hypixelCached.js | 11 | ||||
| -rw-r--r-- | build/index.js | 6 |
6 files changed, 114 insertions, 26 deletions
diff --git a/build/cleaners/rank.js b/build/cleaners/rank.js index 2b8fb77..a9e5f35 100644 --- a/build/cleaners/rank.js +++ b/build/cleaners/rank.js @@ -25,10 +25,12 @@ function cleanRank({ packageRank, newPackageRank, monthlyPackageRank, rankPlusCo name = colored.replace(/ยง./g, '').replace(/[\[\]]/g, ''); } else { - name = monthlyPackageRank - || rank - || (newPackageRank === null || newPackageRank === void 0 ? void 0 : newPackageRank.replace('_PLUS', '+')) - || (packageRank === null || packageRank === void 0 ? void 0 : packageRank.replace('_PLUS', '+')); + if (monthlyPackageRank !== 'NONE') + name = monthlyPackageRank; + else + name = rank + || (newPackageRank === null || newPackageRank === void 0 ? void 0 : newPackageRank.replace('_PLUS', '+')) + || (packageRank === null || packageRank === void 0 ? void 0 : packageRank.replace('_PLUS', '+')); // MVP++ is called Superstar for some reason if (name === 'SUPERSTAR') name = 'MVP++'; diff --git a/build/cleaners/skyblock/profile.js b/build/cleaners/skyblock/profile.js index fe1161e..3a6bc6f 100644 --- a/build/cleaners/skyblock/profile.js +++ b/build/cleaners/skyblock/profile.js @@ -26,13 +26,14 @@ exports.cleanSkyblockProfileResponseLighter = cleanSkyblockProfileResponseLighte * This function is somewhat costly and shouldn't be called often. Use cleanSkyblockProfileResponseLighter if you don't need all the data */ async function cleanSkyblockProfileResponse(data, options) { - const cleanedMembers = []; + // We use Promise.all so it can fetch all the users at once instead of waiting for the previous promise to complete + const promises = []; for (const memberUUID in data.members) { const memberRaw = data.members[memberUUID]; memberRaw.uuid = memberUUID; - const member = await member_1.cleanSkyBlockProfileMemberResponse(memberRaw, ['stats', (options === null || options === void 0 ? void 0 : options.mainMemberUuid) === memberUUID ? 'inventories' : undefined]); - cleanedMembers.push(member); + promises.push(member_1.cleanSkyBlockProfileMemberResponse(memberRaw, ['stats', (options === null || options === void 0 ? void 0 : options.mainMemberUuid) === memberUUID ? 'inventories' : undefined])); } + const cleanedMembers = await Promise.all(promises); const memberMinions = []; for (const member of cleanedMembers) { memberMinions.push(member.minions); diff --git a/build/constants.js b/build/constants.js index 23d2b75..ff6155b 100644 --- a/build/constants.js +++ b/build/constants.js @@ -6,7 +6,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) { return (mod && mod.__esModule) ? mod : { "default": mod }; }; Object.defineProperty(exports, "__esModule", { value: true }); -exports.addStats = exports.fetchStats = void 0; +exports.addCollections = exports.addStats = exports.fetchCollections = exports.fetchStats = void 0; const node_fetch_1 = __importDefault(require("node-fetch")); const https_1 = require("https"); const node_cache_1 = __importDefault(require("node-cache")); @@ -81,6 +81,18 @@ async function fetchStats() { } } exports.fetchStats = fetchStats; +/** Fetch all the known SkyBlock collections as an array of strings */ +async function fetchCollections() { + const file = await fetchFile('collections.json'); + try { + return JSON.parse(file.content); + } + catch { + // probably invalid json, return an empty array + return []; + } +} +exports.fetchCollections = fetchCollections; /** Add stats to skyblock-constants. This has caching so it's fine to call many times */ async function addStats(addingStats) { if (addingStats.length === 0) @@ -109,3 +121,31 @@ async function addStats(addingStats) { await editFile(file, commitMessage, JSON.stringify(updatedStats, null, 2)); } exports.addStats = addStats; +/** Add stats to skyblock-constants. This has caching so it's fine to call many times */ +async function addCollections(addingCollections) { + if (addingCollections.length === 0) + return; // no stats provided, just return + const file = await fetchFile('collections.json'); + if (!file.path) + return; + let oldCollections; + try { + oldCollections = JSON.parse(file.content); + } + catch { + // invalid json, set it as an empty array + oldCollections = []; + } + const updatedCollections = oldCollections + .concat(addingCollections) + // remove duplicates + .filter((value, index, array) => array.indexOf(value) === index) + .sort((a, b) => a.localeCompare(b)); + const newCollections = updatedCollections.filter(value => !oldCollections.includes(value)); + // there's not actually any new stats, just return + if (newCollections.length === 0) + return; + const commitMessage = newCollections.length >= 2 ? `Add ${newCollections.length} new collections` : `Add '${newCollections[0]}'`; + await editFile(file, commitMessage, JSON.stringify(updatedCollections, null, 2)); +} +exports.addCollections = addCollections; diff --git a/build/database.js b/build/database.js index 31c85ee..737b0a3 100644 --- a/build/database.js +++ b/build/database.js @@ -25,8 +25,9 @@ var __importDefault = (this && this.__importDefault) || function (mod) { return (mod && mod.__esModule) ? mod : { "default": mod }; }; Object.defineProperty(exports, "__esModule", { value: true }); -exports.updateDatabaseMember = void 0; +exports.updateDatabaseMember = exports.fetchMemberLeaderboard = void 0; const constants = __importStar(require("./constants")); +const cached = __importStar(require("./hypixelCached")); const mongodb_1 = require("mongodb"); const node_cache_1 = __importDefault(require("node-cache")); // don't update the user for 3 minutes @@ -57,7 +58,7 @@ function getMemberCollectionAttributes(member) { return collectionAttributes; } function getMemberLeaderboardAttributes(member) { - // if you want to add a new leaderboard for member attributes, add it here + // if you want to add a new leaderboard for member attributes, add it here (and getAllLeaderboardAttributes) return { // we use the raw stat names rather than the clean stats in case hypixel adds a new stat and it takes a while for us to clean it ...member.rawHypixelStats, @@ -69,6 +70,19 @@ function getMemberLeaderboardAttributes(member) { visited_zones: member.visited_zones.length, }; } +/** Fetch the names of all the leaderboards */ +async function fetchAllMemberLeaderboardAttributes() { + return [ + // we use the raw stat names rather than the clean stats in case hypixel adds a new stat and it takes a while for us to clean it + ...await constants.fetchStats(), + // collection leaderboards + ...(await constants.fetchCollections()).map(value => `collection_${value}`), + 'fairy_souls', + 'first_join', + 'purse', + 'visited_zones', + ]; +} async function fetchMemberLeaderboard(name) { if (cachedLeaderboards.has(name)) return cachedLeaderboards.get(name); @@ -76,16 +90,28 @@ async function fetchMemberLeaderboard(name) { const query = {}; query[`stats.${name}`] = { '$exists': true }; const sortQuery = {}; - sortQuery[`stats.${name}`] = 1; - const leaderboard = await memberLeaderboardsCollection.find(query).sort(sortQuery).toArray(); + sortQuery[`stats.${name}`] = -1; + const leaderboardRaw = await memberLeaderboardsCollection.find(query).sort(sortQuery).limit(100).toArray(); + const fetchLeaderboardPlayer = async (item) => { + return { + player: await cached.fetchPlayer(item.uuid), + value: item.stats[name] + }; + }; + const promises = []; + for (const item of leaderboardRaw) { + promises.push(fetchLeaderboardPlayer(item)); + } + const leaderboard = await Promise.all(promises); cachedLeaderboards.set(name, leaderboard); return leaderboard; } -async function getLeaderboardRequirement(name) { +exports.fetchMemberLeaderboard = fetchMemberLeaderboard; +async function getMemberLeaderboardRequirement(name) { const leaderboard = await fetchMemberLeaderboard(name); // if there's more than 100 items, return the 100th. if there's less, return null - if (leaderboard.length > 100) - return leaderboard[100].stats[name]; + if (leaderboard.length >= 100) + return leaderboard[99].value; else return null; } @@ -99,6 +125,7 @@ async function updateDatabaseMember(member) { // store the member in recentlyUpdated so it cant update for 3 more minutes recentlyUpdated.set(member.uuid, true); await constants.addStats(Object.keys(member.rawHypixelStats)); + await constants.addCollections(member.collections.map(value => value.name)); const leaderboardAttributes = getMemberLeaderboardAttributes(member); await memberLeaderboardsCollection.updateOne({ uuid: member.uuid @@ -112,4 +139,25 @@ async function updateDatabaseMember(member) { }); } exports.updateDatabaseMember = updateDatabaseMember; -connect(); +/** + * Remove leaderboard attributes for members that wouldn't actually be on the leaderboard. This saves a lot of storage space + */ +async function removeBadMemberLeaderboardAttributes() { + const leaderboards = await fetchAllMemberLeaderboardAttributes(); + for (const leaderboard of leaderboards) { + // wait 10 seconds so it doesnt use as much ram + await new Promise(resolve => setTimeout(resolve, 10000)); + const unsetValue = {}; + unsetValue[leaderboard] = ''; + const filter = {}; + const requirement = await getMemberLeaderboardRequirement(leaderboard); + if (requirement !== null) { + filter[`stats.${leaderboard}`] = { + '$lt': requirement + }; + await memberLeaderboardsCollection.updateMany(filter, { '$unset': unsetValue }); + } + } +} +connect() + .then(removeBadMemberLeaderboardAttributes); diff --git a/build/hypixelCached.js b/build/hypixelCached.js index 5a970f8..67d34d9 100644 --- a/build/hypixelCached.js +++ b/build/hypixelCached.js @@ -81,9 +81,8 @@ async function uuidFromUser(user) { if (usernameCache.has(util_1.undashUuid(user))) { // check if the uuid is a key const username = usernameCache.get(util_1.undashUuid(user)); - // if it has .then, then that means its a waitForSet promise + // if it has .then, then that means its a waitForSet promise. This is done to prevent requests made while it is already requesting if (username.then) { - console.log('pog, prevented double request'); return (await username()).key; } else @@ -92,15 +91,13 @@ async function uuidFromUser(user) { // check if the username is a value const uuidToUsername = usernameCache.mget(usernameCache.keys()); for (const [uuid, username] of Object.entries(uuidToUsername)) { - if (user.toLowerCase() === username.toLowerCase()) + if (username.toLowerCase && user.toLowerCase() === username.toLowerCase()) return uuid; } if (_1.debug) console.log('Cache miss: uuidFromUser', user); // set it as waitForSet (a promise) in case uuidFromUser gets called while its fetching mojang - console.log('setting', util_1.undashUuid(user)); usernameCache.set(util_1.undashUuid(user), waitForSet(usernameCache, user, user)); - console.log(util_1.undashUuid(user), usernameCache.has(util_1.undashUuid(user))); // not cached, actually fetch mojang api now let { uuid, username } = await mojang.mojangDataFromUser(user); if (!uuid) @@ -134,12 +131,8 @@ exports.usernameFromUser = usernameFromUser; async function fetchPlayer(user) { const playerUuid = await uuidFromUser(user); if (playerCache.has(playerUuid)) { - if (_1.debug) - console.log('Cache hit! fetchPlayer', user); return playerCache.get(playerUuid); } - if (_1.debug) - console.log('Cache miss: uuidFromUser', user); const cleanPlayer = await hypixel.sendCleanApiRequest({ path: 'player', args: { uuid: playerUuid } diff --git a/build/index.js b/build/index.js index 3174e69..52e711c 100644 --- a/build/index.js +++ b/build/index.js @@ -6,8 +6,9 @@ Object.defineProperty(exports, "__esModule", { value: true }); exports.debug = void 0; const hypixel_1 = require("./hypixel"); const express_1 = __importDefault(require("express")); +const database_1 = require("./database"); const app = express_1.default(); -exports.debug = true; +exports.debug = false; app.use((req, res, next) => { if (process.env.key && req.headers.key !== process.env.key) // if a key is set in process.env and the header doesn't match return an error @@ -24,4 +25,7 @@ app.get('/player/:user', async (req, res) => { app.get('/player/:user/:profile', async (req, res) => { res.json(await hypixel_1.fetchMemberProfile(req.params.user, req.params.profile)); }); +app.get('/leaderboard/:name', async (req, res) => { + res.json(await database_1.fetchMemberLeaderboard(req.params.name)); +}); app.listen(8080, () => console.log('App started :)')); |