diff options
Diffstat (limited to 'build')
-rw-r--r-- | build/cleaners/skyblock/inventory.js | 1 | ||||
-rw-r--r-- | build/cleaners/skyblock/member.js | 2 | ||||
-rw-r--r-- | build/cleaners/skyblock/profile.js | 12 | ||||
-rw-r--r-- | build/database.js | 184 | ||||
-rw-r--r-- | build/hypixel.js | 17 | ||||
-rw-r--r-- | build/hypixelCached.js | 30 | ||||
-rw-r--r-- | build/index.js | 8 |
7 files changed, 234 insertions, 20 deletions
diff --git a/build/cleaners/skyblock/inventory.js b/build/cleaners/skyblock/inventory.js index 0131e1b..400408c 100644 --- a/build/cleaners/skyblock/inventory.js +++ b/build/cleaners/skyblock/inventory.js @@ -20,6 +20,7 @@ var __importStar = (this && this.__importStar) || function (mod) { }; Object.defineProperty(exports, "__esModule", { value: true }); exports.cleanInventories = exports.INVENTORIES = exports.cleanInventory = void 0; +// maybe todo?: create a fast replacement for prismarine-nbt const nbt = __importStar(require("prismarine-nbt")); function base64decode(base64) { return Buffer.from(base64, 'base64'); diff --git a/build/cleaners/skyblock/member.js b/build/cleaners/skyblock/member.js index 9869646..ce322bc 100644 --- a/build/cleaners/skyblock/member.js +++ b/build/cleaners/skyblock/member.js @@ -45,7 +45,7 @@ exports.cleanSkyBlockProfileMemberResponseBasic = cleanSkyBlockProfileMemberResp async function cleanSkyBlockProfileMemberResponse(member, included = null) { var _a; // profiles.members[] - const inventoriesIncluded = included == null || included.includes('inventories'); + const inventoriesIncluded = included === null || included.includes('inventories'); const player = await cached.fetchPlayer(member.uuid); if (!player) return; diff --git a/build/cleaners/skyblock/profile.js b/build/cleaners/skyblock/profile.js index ef90ce8..ba2c030 100644 --- a/build/cleaners/skyblock/profile.js +++ b/build/cleaners/skyblock/profile.js @@ -31,9 +31,19 @@ async function cleanSkyblockProfileResponse(data, options) { for (const memberUUID in data.members) { const memberRaw = data.members[memberUUID]; memberRaw.uuid = memberUUID; - promises.push(member_1.cleanSkyBlockProfileMemberResponse(memberRaw, ['stats', (options === null || options === void 0 ? void 0 : options.mainMemberUuid) === memberUUID ? 'inventories' : undefined])); + promises.push(member_1.cleanSkyBlockProfileMemberResponse(memberRaw, [ + !(options === null || options === void 0 ? void 0 : options.basic) ? 'stats' : undefined, + (options === null || options === void 0 ? void 0 : options.mainMemberUuid) === memberUUID ? 'inventories' : undefined + ])); } const cleanedMembers = (await Promise.all(promises)).filter(m => m !== null && m !== undefined); + if (options === null || options === void 0 ? void 0 : options.basic) { + return { + uuid: data.profile_id, + name: data.cute_name, + members: cleanedMembers, + }; + } const memberMinions = []; for (const member of cleanedMembers) { memberMinions.push(member.minions); diff --git a/build/database.js b/build/database.js index be193e9..600b21d 100644 --- a/build/database.js +++ b/build/database.js @@ -25,7 +25,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) { return (mod && mod.__esModule) ? mod : { "default": mod }; }; Object.defineProperty(exports, "__esModule", { value: true }); -exports.queueUpdateDatabaseMember = exports.updateDatabaseMember = exports.fetchMemberLeaderboardSpots = exports.fetchMemberLeaderboard = exports.fetchAllMemberLeaderboardAttributes = exports.fetchSlayerLeaderboards = exports.fetchAllLeaderboardsCategorized = void 0; +exports.queueUpdateDatabaseProfile = exports.queueUpdateDatabaseMember = exports.updateDatabaseProfile = exports.updateDatabaseMember = exports.fetchMemberLeaderboardSpots = exports.fetchLeaderboard = exports.fetchProfileLeaderboard = exports.fetchMemberLeaderboard = exports.fetchAllMemberLeaderboardAttributes = exports.fetchSlayerLeaderboards = exports.fetchAllLeaderboardsCategorized = void 0; const stats_1 = require("./cleaners/skyblock/stats"); const mongodb_1 = require("mongodb"); const cached = __importStar(require("./hypixelCached")); @@ -50,6 +50,7 @@ const reversedLeaderboards = [ let client; let database; let memberLeaderboardsCollection; +let profileLeaderboardsCollection; async function connect() { if (!process.env.db_uri) return console.warn('Warning: db_uri was not found in .env. Features that utilize the database such as leaderboards won\'t work.'); @@ -58,6 +59,7 @@ async function connect() { client = await mongodb_1.MongoClient.connect(process.env.db_uri, { useNewUrlParser: true, useUnifiedTopology: true }); database = client.db(process.env.db_name); memberLeaderboardsCollection = database.collection('member-leaderboards'); + profileLeaderboardsCollection = database.collection('profile-leaderboards'); } function getMemberCollectionAttributes(member) { const collectionAttributes = {}; @@ -106,10 +108,17 @@ function getMemberLeaderboardAttributes(member) { visited_zones: member.visited_zones.length, }; } +function getProfileLeaderboardAttributes(profile) { + // if you want to add a new leaderboard for member attributes, add it here (and getAllLeaderboardAttributes) + return { + minion_count: profile.minion_count + }; +} async function fetchAllLeaderboardsCategorized() { const memberLeaderboardAttributes = await fetchAllMemberLeaderboardAttributes(); + const profileLeaderboardAttributes = await fetchAllProfileLeaderboardAttributes(); const categorizedLeaderboards = {}; - for (const leaderboard of memberLeaderboardAttributes) { + for (const leaderboard of [...memberLeaderboardAttributes, ...profileLeaderboardAttributes]) { const { category } = stats_1.categorizeStat(leaderboard); if (!categorizedLeaderboards[category]) categorizedLeaderboards[category] = []; @@ -140,7 +149,7 @@ async function fetchSlayerLeaderboards() { return leaderboardNames; } exports.fetchSlayerLeaderboards = fetchSlayerLeaderboards; -/** Fetch the names of all the leaderboards */ +/** Fetch the names of all the leaderboards that rank members */ async function fetchAllMemberLeaderboardAttributes() { return [ // we use the raw stat names rather than the clean stats in case hypixel adds a new stat and it takes a while for us to clean it @@ -159,6 +168,12 @@ async function fetchAllMemberLeaderboardAttributes() { ]; } exports.fetchAllMemberLeaderboardAttributes = fetchAllMemberLeaderboardAttributes; +/** Fetch the names of all the leaderboards that rank profiles */ +async function fetchAllProfileLeaderboardAttributes() { + return [ + 'minion_count' + ]; +} function isLeaderboardReversed(name) { for (const leaderboardMatch of reversedLeaderboards) { let trailingEnd = leaderboardMatch[0] === '_'; @@ -171,6 +186,8 @@ function isLeaderboardReversed(name) { return false; } async function fetchMemberLeaderboardRaw(name) { + if (!client) + throw Error('Client isn\'t initialized yet'); if (cachedRawLeaderboards.has(name)) return cachedRawLeaderboards.get(name); // typescript forces us to make a new variable and set it this way because it gives an error otherwise @@ -186,6 +203,23 @@ async function fetchMemberLeaderboardRaw(name) { cachedRawLeaderboards.set(name, leaderboardRaw); return leaderboardRaw; } +async function fetchProfileLeaderboardRaw(name) { + if (cachedRawLeaderboards.has(name)) + return cachedRawLeaderboards.get(name); + // typescript forces us to make a new variable and set it this way because it gives an error otherwise + const query = {}; + query[`stats.${name}`] = { '$exists': true, '$ne': NaN }; + const sortQuery = {}; + sortQuery[`stats.${name}`] = isLeaderboardReversed(name) ? 1 : -1; + const leaderboardRaw = await profileLeaderboardsCollection + .find(query) + .sort(sortQuery) + .limit(leaderboardMax) + .toArray(); + console.log('leaderboardRaw', leaderboardRaw); + cachedRawLeaderboards.set(name, leaderboardRaw); + return leaderboardRaw; +} /** Fetch a leaderboard that ranks members, as opposed to profiles */ async function fetchMemberLeaderboard(name) { var _a; @@ -193,6 +227,7 @@ async function fetchMemberLeaderboard(name) { const fetchLeaderboardPlayer = async (item) => { return { player: await cached.fetchBasicPlayer(item.uuid), + profileUuid: item.profile, value: item.stats[name] }; }; @@ -208,6 +243,44 @@ async function fetchMemberLeaderboard(name) { }; } exports.fetchMemberLeaderboard = fetchMemberLeaderboard; +/** Fetch a leaderboard that ranks profiles, as opposed to members */ +async function fetchProfileLeaderboard(name) { + var _a; + const leaderboardRaw = await fetchProfileLeaderboardRaw(name); + const fetchLeaderboardProfile = async (item) => { + const players = []; + for (const playerUuid of item.players) + players.push(await cached.fetchBasicPlayer(playerUuid)); + return { + players: players, + profileUuid: item.uuid, + value: item.stats[name] + }; + }; + const promises = []; + for (const item of leaderboardRaw) { + promises.push(fetchLeaderboardProfile(item)); + } + const leaderboard = await Promise.all(promises); + return { + name: name, + unit: (_a = stats_1.getStatUnit(name)) !== null && _a !== void 0 ? _a : null, + list: leaderboard + }; +} +exports.fetchProfileLeaderboard = fetchProfileLeaderboard; +/** Fetch a leaderboard */ +async function fetchLeaderboard(name) { + const profileLeaderboards = await fetchAllProfileLeaderboardAttributes(); + console.log(name, profileLeaderboards, profileLeaderboards.includes(name)); + if (profileLeaderboards.includes(name)) { + return await fetchProfileLeaderboard(name); + } + else { + return await fetchMemberLeaderboard(name); + } +} +exports.fetchLeaderboard = fetchLeaderboard; /** Get the leaderboard positions a member is on. This may take a while depending on whether stuff is cached */ async function fetchMemberLeaderboardSpots(player, profile) { var _a; @@ -215,7 +288,7 @@ async function fetchMemberLeaderboardSpots(player, profile) { const fullMember = fullProfile.members.find(m => m.username.toLowerCase() === player.toLowerCase() || m.uuid === player); // update the leaderboard positions for the member await updateDatabaseMember(fullMember, fullProfile); - const applicableAttributes = await getApplicableAttributes(fullMember); + const applicableAttributes = await getApplicableMemberLeaderboardAttributes(fullMember); const memberLeaderboardSpots = []; for (const leaderboardName in applicableAttributes) { const leaderboard = await fetchMemberLeaderboardRaw(leaderboardName); @@ -230,8 +303,12 @@ async function fetchMemberLeaderboardSpots(player, profile) { return memberLeaderboardSpots; } exports.fetchMemberLeaderboardSpots = fetchMemberLeaderboardSpots; -async function getMemberLeaderboardRequirement(name) { - const leaderboard = await fetchMemberLeaderboardRaw(name); +async function getLeaderboardRequirement(name, leaderboardType) { + let leaderboard; + if (leaderboardType === 'member') + leaderboard = await fetchMemberLeaderboardRaw(name); + else if (leaderboardType === 'profile') + leaderboard = await fetchProfileLeaderboardRaw(name); // if there's more than 100 items, return the 100th. if there's less, return null if (leaderboard.length >= leaderboardMax) return leaderboard[leaderboardMax - 1].stats[name]; @@ -239,11 +316,31 @@ async function getMemberLeaderboardRequirement(name) { return null; } /** Get the attributes for the member, but only ones that would put them on the top 100 for leaderboards */ -async function getApplicableAttributes(member) { +async function getApplicableMemberLeaderboardAttributes(member) { const leaderboardAttributes = getMemberLeaderboardAttributes(member); const applicableAttributes = {}; for (const [leaderboard, attributeValue] of Object.entries(leaderboardAttributes)) { - const requirement = await getMemberLeaderboardRequirement(leaderboard); + const requirement = await getLeaderboardRequirement(leaderboard, 'member'); + const leaderboardReversed = isLeaderboardReversed(leaderboard); + if ((requirement === null) + || (leaderboardReversed ? attributeValue < requirement : attributeValue > requirement)) { + applicableAttributes[leaderboard] = attributeValue; + } + } + let leaderboardsCount = Object.keys(applicableAttributes).length; + const leaderboardsCountRequirement = await getLeaderboardRequirement('leaderboards_count', 'member'); + if ((leaderboardsCountRequirement === null) + || (leaderboardsCount > leaderboardsCountRequirement)) { + applicableAttributes['leaderboards_count'] = leaderboardsCount; + } + return applicableAttributes; +} +/** Get the attributes for the profile, but only ones that would put them on the top 100 for leaderboards */ +async function getApplicableProfileLeaderboardAttributes(profile) { + const leaderboardAttributes = getProfileLeaderboardAttributes(profile); + const applicableAttributes = {}; + for (const [leaderboard, attributeValue] of Object.entries(leaderboardAttributes)) { + const requirement = await getLeaderboardRequirement(leaderboard, 'profile'); const leaderboardReversed = isLeaderboardReversed(leaderboard); if ((requirement === null) || (leaderboardReversed ? attributeValue < requirement : attributeValue > requirement)) { @@ -251,10 +348,9 @@ async function getApplicableAttributes(member) { } } let leaderboardsCount = Object.keys(applicableAttributes).length; - const leaderboardsCountRequirement = await getMemberLeaderboardRequirement('leaderboards_count'); + const leaderboardsCountRequirement = await getLeaderboardRequirement('leaderboards_count', 'member'); if ((leaderboardsCountRequirement === null) || (leaderboardsCount > leaderboardsCountRequirement)) { - // add 1 extra because this attribute also counts :) applicableAttributes['leaderboards_count'] = leaderboardsCount; } return applicableAttributes; @@ -279,9 +375,9 @@ async function updateDatabaseMember(member, profile) { await constants.addSlayers(member.slayers.bosses.map(s => s.raw_name)); if (_1.debug) console.log('done constants..'); - const leaderboardAttributes = await getApplicableAttributes(member); + const leaderboardAttributes = await getApplicableMemberLeaderboardAttributes(member); if (_1.debug) - console.log('done getApplicableAttributes..', leaderboardAttributes, member.username, profile.name); + console.log('done getApplicableMemberLeaderboardAttributes..', leaderboardAttributes, member.username, profile.name); await memberLeaderboardsCollection.updateOne({ uuid: member.uuid, profile: profile.uuid @@ -311,15 +407,73 @@ async function updateDatabaseMember(member, profile) { console.log('added member to leaderboards', member.username, leaderboardAttributes); } exports.updateDatabaseMember = updateDatabaseMember; -const leaderboardUpdateQueue = new queue_promise_1.default({ +/** + * Update the profiles's leaderboard data on the server if applicable. + * This will not also update the members, you have to call updateDatabaseMember separately for that + */ +async function updateDatabaseProfile(profile) { + if (_1.debug) + console.log('updateDatabaseProfile', profile.name); + if (!client) + return; // the db client hasn't been initialized + // the profile's been updated too recently, just return + if (recentlyUpdated.get(profile.uuid + 'profile')) + return; + // store the profile in recentlyUpdated so it cant update for 3 more minutes + recentlyUpdated.set(profile.uuid + 'profile', true); + if (_1.debug) + console.log('adding profile to leaderboards', profile.name); + const leaderboardAttributes = await getApplicableProfileLeaderboardAttributes(profile); + if (_1.debug) + console.log('done getApplicableProfileLeaderboardAttributes..', leaderboardAttributes, profile.name); + await profileLeaderboardsCollection.updateOne({ + uuid: profile.uuid + }, { + '$set': { + players: profile.members.map(p => p.uuid), + stats: leaderboardAttributes, + last_updated: new Date() + } + }, { upsert: true }); + // add the profile to the cached leaderboard without having to refetch it + for (const [attributeName, attributeValue] of Object.entries(leaderboardAttributes)) { + const existingRawLeaderboard = await fetchProfileLeaderboardRaw(attributeName); + const leaderboardReverse = isLeaderboardReversed(attributeName); + const newRawLeaderboard = existingRawLeaderboard + // remove the player from the leaderboard, if they're there + .filter(value => value.uuid !== profile.uuid) + .concat([{ + last_updated: new Date(), + stats: leaderboardAttributes, + uuid: profile.uuid, + players: profile.members.map(p => p.uuid) + }]) + .sort((a, b) => leaderboardReverse ? a.stats[attributeName] - b.stats[attributeName] : b.stats[attributeName] - a.stats[attributeName]) + .slice(0, 100); + cachedRawLeaderboards.set(attributeName, newRawLeaderboard); + } + if (_1.debug) + console.log('added profile to leaderboards', profile.name, leaderboardAttributes); +} +exports.updateDatabaseProfile = updateDatabaseProfile; +const leaderboardUpdateMemberQueue = new queue_promise_1.default({ concurrent: 1, interval: 500 }); +const leaderboardUpdateProfileQueue = new queue_promise_1.default({ + concurrent: 1, + interval: 2000 +}); /** Queue an update for the member's leaderboard data on the server if applicable */ async function queueUpdateDatabaseMember(member, profile) { - leaderboardUpdateQueue.enqueue(async () => await updateDatabaseMember(member, profile)); + leaderboardUpdateMemberQueue.enqueue(async () => await updateDatabaseMember(member, profile)); } exports.queueUpdateDatabaseMember = queueUpdateDatabaseMember; +/** Queue an update for the profile's leaderboard data on the server if applicable */ +async function queueUpdateDatabaseProfile(profile) { + leaderboardUpdateProfileQueue.enqueue(async () => await updateDatabaseProfile(profile)); +} +exports.queueUpdateDatabaseProfile = queueUpdateDatabaseProfile; /** * Remove leaderboard attributes for members that wouldn't actually be on the leaderboard. This saves a lot of storage space */ @@ -332,7 +486,7 @@ async function removeBadMemberLeaderboardAttributes() { const unsetValue = {}; unsetValue[leaderboard] = ''; const filter = {}; - const requirement = await getMemberLeaderboardRequirement(leaderboard); + const requirement = await getLeaderboardRequirement(leaderboard, 'member'); const leaderboardReversed = isLeaderboardReversed(leaderboard); if (requirement !== null) { filter[`stats.${leaderboard}`] = { diff --git a/build/hypixel.js b/build/hypixel.js index 175ee79..9e9007e 100644 --- a/build/hypixel.js +++ b/build/hypixel.js @@ -22,7 +22,7 @@ var __importStar = (this && this.__importStar) || function (mod) { return result; }; Object.defineProperty(exports, "__esModule", { value: true }); -exports.fetchMemberProfilesUncached = exports.fetchMemberProfileUncached = exports.fetchMemberProfile = exports.fetchUser = exports.sendCleanApiRequest = exports.maxMinion = exports.saveInterval = void 0; +exports.fetchMemberProfilesUncached = exports.fetchBasicProfileFromUuidUncached = exports.fetchMemberProfileUncached = exports.fetchMemberProfile = exports.fetchUser = exports.sendCleanApiRequest = exports.maxMinion = exports.saveInterval = void 0; const player_1 = require("./cleaners/player"); const hypixelApi_1 = require("./hypixelApi"); const cached = __importStar(require("./hypixelCached")); @@ -159,9 +159,23 @@ async function fetchMemberProfileUncached(playerUuid, profileUuid) { // queue updating the leaderboard positions for the member, eventually for (const member of profile.members) database_1.queueUpdateDatabaseMember(member, profile); + database_1.queueUpdateDatabaseProfile(profile); return profile; } exports.fetchMemberProfileUncached = fetchMemberProfileUncached; +/** + * Fetches the Hypixel API to get a CleanProfile from its id. This doesn't do any caching and you should use hypixelCached.fetchBasicProfileFromUuid instead + * @param playerUuid The UUID of the Minecraft player + * @param profileUuid The UUID of the Hypixel SkyBlock profile + */ +async function fetchBasicProfileFromUuidUncached(profileUuid) { + const profile = await sendCleanApiRequest({ + path: 'skyblock/profile', + args: { profile: profileUuid } + }, null, { basic: true }); + return profile; +} +exports.fetchBasicProfileFromUuidUncached = fetchBasicProfileFromUuidUncached; async function fetchMemberProfilesUncached(playerUuid) { const profiles = await sendCleanApiRequest({ path: 'skyblock/profiles', @@ -176,6 +190,7 @@ async function fetchMemberProfilesUncached(playerUuid) { for (const member of profile.members) { database_1.queueUpdateDatabaseMember(member, profile); } + database_1.queueUpdateDatabaseProfile(profile); } return profiles; } diff --git a/build/hypixelCached.js b/build/hypixelCached.js index 8693032..cad0e94 100644 --- a/build/hypixelCached.js +++ b/build/hypixelCached.js @@ -25,7 +25,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) { return (mod && mod.__esModule) ? mod : { "default": mod }; }; Object.defineProperty(exports, "__esModule", { value: true }); -exports.fetchProfileName = exports.fetchProfile = exports.fetchProfileUuid = exports.fetchSkyblockProfiles = exports.fetchBasicPlayer = exports.fetchPlayer = exports.usernameFromUser = exports.uuidFromUser = exports.profileNameCache = exports.profilesCache = exports.profileCache = exports.basicPlayerCache = exports.playerCache = exports.basicProfilesCache = exports.usernameCache = void 0; +exports.fetchProfileName = exports.fetchBasicProfileFromUuid = exports.fetchProfile = exports.fetchProfileUuid = exports.fetchSkyblockProfiles = exports.fetchBasicPlayer = exports.fetchPlayer = exports.usernameFromUser = exports.uuidFromUser = exports.profileNameCache = exports.profilesCache = exports.profileCache = exports.basicPlayerCache = exports.playerCache = exports.basicProfilesCache = exports.usernameCache = void 0; const node_cache_1 = __importDefault(require("node-cache")); const mojang = __importStar(require("./mojang")); const hypixel = __importStar(require("./hypixel")); @@ -180,6 +180,8 @@ async function fetchBasicPlayer(user) { if (exports.basicPlayerCache.has(playerUuid)) return exports.basicPlayerCache.get(playerUuid); const player = await fetchPlayer(playerUuid); + if (!player) + console.log(user); delete player.profiles; return player; } @@ -287,6 +289,32 @@ async function fetchProfile(user, profile) { } exports.fetchProfile = fetchProfile; /** + * Fetch a CleanProfile from the uuid + * @param profileUuid A profile name or profile uuid +*/ +async function fetchBasicProfileFromUuid(profileUuid) { + if (exports.profileCache.has(profileUuid)) { + // we have the profile cached, return it :) + if (_1.debug) + console.log('Cache hit! fetchBasicProfileFromUuid', profileUuid); + const profile = exports.profileCache.get(profileUuid); + return { + uuid: profile.uuid, + members: profile.members.map(m => ({ + uuid: m.uuid, + username: m.username, + last_save: m.last_save, + first_join: m.first_join, + rank: m.rank, + })), + name: profile.name + }; + } + // TODO: cache this + return await hypixel.fetchBasicProfileFromUuidUncached(profileUuid); +} +exports.fetchBasicProfileFromUuid = fetchBasicProfileFromUuid; +/** * Fetch the name of a profile from the user and profile uuid * @param user A player uuid or username * @param profile A profile uuid or name diff --git a/build/index.js b/build/index.js index 30d2553..2d7ab25 100644 --- a/build/index.js +++ b/build/index.js @@ -40,7 +40,13 @@ app.get('/player/:user/:profile/leaderboards', async (req, res) => { res.json(await database_1.fetchMemberLeaderboardSpots(req.params.user, req.params.profile)); }); app.get('/leaderboard/:name', async (req, res) => { - res.json(await database_1.fetchMemberLeaderboard(req.params.name)); + try { + res.json(await database_1.fetchLeaderboard(req.params.name)); + } + catch (err) { + console.error(err); + res.json({ 'error': err.toString() }); + } }); app.get('/leaderboards', async (req, res) => { res.json(await database_1.fetchAllLeaderboardsCategorized()); |