diff options
| -rw-r--r-- | build/database.js | 69 | ||||
| -rw-r--r-- | build/hypixelCached.js | 24 | ||||
| -rw-r--r-- | src/database.ts | 70 | ||||
| -rw-r--r-- | src/hypixelCached.ts | 25 |
4 files changed, 153 insertions, 35 deletions
diff --git a/build/database.js b/build/database.js index d28d934..05a082f 100644 --- a/build/database.js +++ b/build/database.js @@ -33,6 +33,7 @@ const node_cache_1 = __importDefault(require("node-cache")); const util_1 = require("./util"); const stats_1 = require("./cleaners/skyblock/stats"); const queue_promise_1 = __importDefault(require("queue-promise")); +const _1 = require("."); // don't update the user for 3 minutes const recentlyUpdated = new node_cache_1.default({ stdTTL: 60 * 3, @@ -41,10 +42,14 @@ const recentlyUpdated = new node_cache_1.default({ }); const cachedRawLeaderboards = new Map(); const leaderboardMax = 100; -const reversedStats = [ +const reversedLeaderboards = [ 'first_join', '_best_time', '_best_time_2' ]; +const leaderboardUnits = { + time: ['_best_time', '_best_time_2'], + date: ['first_join'] +}; let client; let database; let memberLeaderboardsCollection; @@ -121,16 +126,28 @@ async function fetchAllMemberLeaderboardAttributes() { } exports.fetchAllMemberLeaderboardAttributes = fetchAllMemberLeaderboardAttributes; function isLeaderboardReversed(name) { - for (const statMatch of reversedStats) { - let trailingEnd = statMatch[0] === '_'; - let trailingStart = statMatch.substr(-1) === '_'; - if ((trailingStart && name.startsWith(statMatch)) - || (trailingEnd && name.endsWith(statMatch)) - || (name == statMatch)) + for (const leaderboardMatch of reversedLeaderboards) { + let trailingEnd = leaderboardMatch[0] === '_'; + let trailingStart = leaderboardMatch.substr(-1) === '_'; + if ((trailingStart && name.startsWith(leaderboardMatch)) + || (trailingEnd && name.endsWith(leaderboardMatch)) + || (name == leaderboardMatch)) return true; } return false; } +function getLeaderboardUnit(name) { + for (const [unitName, leaderboardMatchers] of Object.entries(leaderboardUnits)) { + for (const leaderboardMatch of leaderboardMatchers) { + let trailingEnd = leaderboardMatch[0] === '_'; + let trailingStart = leaderboardMatch.substr(-1) === '_'; + if ((trailingStart && name.startsWith(leaderboardMatch)) + || (trailingEnd && name.endsWith(leaderboardMatch)) + || (name == leaderboardMatch)) + return unitName; + } + } +} async function fetchMemberLeaderboardRaw(name) { if (cachedRawLeaderboards.has(name)) return cachedRawLeaderboards.get(name); @@ -144,10 +161,11 @@ async function fetchMemberLeaderboardRaw(name) { return leaderboardRaw; } async function fetchMemberLeaderboard(name) { + var _a; const leaderboardRaw = await fetchMemberLeaderboardRaw(name); const fetchLeaderboardPlayer = async (item) => { return { - player: await cached.fetchPlayer(item.uuid), + player: await cached.fetchBasicPlayer(item.uuid), value: item.stats[name] }; }; @@ -156,7 +174,11 @@ async function fetchMemberLeaderboard(name) { promises.push(fetchLeaderboardPlayer(item)); } const leaderboard = await Promise.all(promises); - return leaderboard; + return { + name: name, + unit: (_a = getLeaderboardUnit(name)) !== null && _a !== void 0 ? _a : null, + list: leaderboard + }; } exports.fetchMemberLeaderboard = fetchMemberLeaderboard; async function getMemberLeaderboardRequirement(name) { @@ -175,13 +197,11 @@ async function getApplicableAttributes(member) { const requirement = await getMemberLeaderboardRequirement(attributeName); if (!requirement || attributeValue > requirement) applicableAttributes[attributeName] = attributeValue; - console.log(attributeName); } return applicableAttributes; } /** Update the member's leaderboard data on the server if applicable */ async function updateDatabaseMember(member, profile) { - console.log('updating', member.uuid); if (!client) return; // the db client hasn't been initialized // the member's been updated too recently, just return @@ -218,7 +238,6 @@ async function updateDatabaseMember(member, profile) { .slice(0, 100); cachedRawLeaderboards.set(attributeName, newRawLeaderboard); } - console.log('updated', member.uuid); } exports.updateDatabaseMember = updateDatabaseMember; const queue = new queue_promise_1.default({ @@ -238,7 +257,7 @@ async function removeBadMemberLeaderboardAttributes() { // shuffle so if the application is restarting many times itll still be useful for (const leaderboard of util_1.shuffle(leaderboards)) { // wait 10 seconds so it doesnt use as much ram - await util_1.sleep(100000); + await util_1.sleep(10 * 1000); const unsetValue = {}; unsetValue[leaderboard] = ''; const filter = {}; @@ -251,5 +270,25 @@ async function removeBadMemberLeaderboardAttributes() { } } } -connect() - .then(removeBadMemberLeaderboardAttributes); +/** Fetch all the leaderboards, used for caching. Don't call this often! */ +async function fetchAllLeaderboards() { + const leaderboards = await fetchAllMemberLeaderboardAttributes(); + // shuffle so if the application is restarting many times itll still be useful + if (_1.debug) + console.log('Caching leaderboards!'); + for (const leaderboard of util_1.shuffle(leaderboards)) { + // wait 2 seconds so it doesnt use as much ram + await util_1.sleep(2 * 1000); + await fetchMemberLeaderboard(leaderboard); + } + if (_1.debug) + console.log('Finished caching leaderboards!'); +} +connect().then(() => { + // when it connects, cache the leaderboards and remove bad members + removeBadMemberLeaderboardAttributes(); + // cache leaderboards on startup so its faster later on + fetchAllLeaderboards(); + // cache leaderboard players again every hour + setInterval(fetchAllLeaderboards, 4 * 60 * 60 * 1000); +}); diff --git a/build/hypixelCached.js b/build/hypixelCached.js index ccb0ca6..87f4a9e 100644 --- a/build/hypixelCached.js +++ b/build/hypixelCached.js @@ -25,7 +25,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) { return (mod && mod.__esModule) ? mod : { "default": mod }; }; Object.defineProperty(exports, "__esModule", { value: true }); -exports.fetchProfileName = exports.fetchProfile = exports.fetchProfileUuid = exports.fetchSkyblockProfiles = exports.fetchPlayer = exports.usernameFromUser = exports.uuidFromUser = void 0; +exports.fetchProfileName = exports.fetchProfile = exports.fetchProfileUuid = exports.fetchSkyblockProfiles = exports.fetchBasicPlayer = exports.fetchPlayer = exports.usernameFromUser = exports.uuidFromUser = void 0; const node_cache_1 = __importDefault(require("node-cache")); const mojang = __importStar(require("./mojang")); const hypixel = __importStar(require("./hypixel")); @@ -47,6 +47,12 @@ const playerCache = new node_cache_1.default({ checkperiod: 10, useClones: true, }); +// cache "basic players" (players without profiles) for 4 hours +const basicPlayerCache = new node_cache_1.default({ + stdTTL: 60 * 60 * 4, + checkperiod: 60 * 10, + useClones: true +}); const profileCache = new node_cache_1.default({ stdTTL: 30, checkperiod: 10, @@ -137,9 +143,8 @@ async function usernameFromUser(user) { exports.usernameFromUser = usernameFromUser; async function fetchPlayer(user) { const playerUuid = await uuidFromUser(user); - if (playerCache.has(playerUuid)) { + if (playerCache.has(playerUuid)) return playerCache.get(playerUuid); - } const cleanPlayer = await hypixel.sendCleanApiRequest({ path: 'player', args: { uuid: playerUuid } @@ -149,9 +154,22 @@ async function fetchPlayer(user) { // clone in case it gets modified somehow later playerCache.set(playerUuid, cleanPlayer); usernameCache.set(playerUuid, cleanPlayer.username); + const cleanBasicPlayer = Object.assign({}, cleanPlayer); + delete cleanBasicPlayer.profiles; + basicPlayerCache.set(playerUuid, cleanBasicPlayer); return cleanPlayer; } exports.fetchPlayer = fetchPlayer; +/** Fetch a player without their profiles. This is heavily cached. */ +async function fetchBasicPlayer(user) { + const playerUuid = await uuidFromUser(user); + if (basicPlayerCache.has(playerUuid)) + return basicPlayerCache.get(playerUuid); + const player = await fetchPlayer(playerUuid); + delete player.profiles; + return player; +} +exports.fetchBasicPlayer = fetchBasicPlayer; async function fetchSkyblockProfiles(playerUuid) { if (profilesCache.has(playerUuid)) { if (_1.debug) diff --git a/src/database.ts b/src/database.ts index 29760ef..466e2f1 100644 --- a/src/database.ts +++ b/src/database.ts @@ -12,6 +12,7 @@ import { shuffle, sleep } from './util' import { CleanFullProfile } from './cleaners/skyblock/profile' import { categorizeStat } from './cleaners/skyblock/stats' import Queue from 'queue-promise' +import { debug } from '.' // don't update the user for 3 minutes const recentlyUpdated = new NodeCache({ @@ -34,10 +35,14 @@ interface LeaderboardItem { const cachedRawLeaderboards: Map<string, DatabaseLeaderboardItem[]> = new Map() const leaderboardMax = 100 -const reversedStats = [ +const reversedLeaderboards = [ 'first_join', '_best_time', '_best_time_2' ] +const leaderboardUnits = { + time: ['_best_time', '_best_time_2'], + date: ['first_join'] +} let client: MongoClient let database: Db @@ -131,19 +136,34 @@ export async function fetchAllMemberLeaderboardAttributes(): Promise<string[]> { } function isLeaderboardReversed(name: string): boolean { - for (const statMatch of reversedStats) { - let trailingEnd = statMatch[0] === '_' - let trailingStart = statMatch.substr(-1) === '_' + for (const leaderboardMatch of reversedLeaderboards) { + let trailingEnd = leaderboardMatch[0] === '_' + let trailingStart = leaderboardMatch.substr(-1) === '_' if ( - (trailingStart && name.startsWith(statMatch)) - || (trailingEnd && name.endsWith(statMatch)) - || (name == statMatch) + (trailingStart && name.startsWith(leaderboardMatch)) + || (trailingEnd && name.endsWith(leaderboardMatch)) + || (name == leaderboardMatch) ) return true } return false } +function getLeaderboardUnit(name: string): string { + for (const [ unitName, leaderboardMatchers ] of Object.entries(leaderboardUnits)) { + for (const leaderboardMatch of leaderboardMatchers) { + let trailingEnd = leaderboardMatch[0] === '_' + let trailingStart = leaderboardMatch.substr(-1) === '_' + if ( + (trailingStart && name.startsWith(leaderboardMatch)) + || (trailingEnd && name.endsWith(leaderboardMatch)) + || (name == leaderboardMatch) + ) + return unitName + } + } +} + async function fetchMemberLeaderboardRaw(name: string): Promise<DatabaseLeaderboardItem[]> { if (cachedRawLeaderboards.has(name)) return cachedRawLeaderboards.get(name) @@ -163,7 +183,7 @@ export async function fetchMemberLeaderboard(name: string) { const leaderboardRaw = await fetchMemberLeaderboardRaw(name) const fetchLeaderboardPlayer = async(item: DatabaseLeaderboardItem): Promise<LeaderboardItem> => { return { - player: await cached.fetchPlayer(item.uuid), + player: await cached.fetchBasicPlayer(item.uuid), value: item.stats[name] } } @@ -172,7 +192,11 @@ export async function fetchMemberLeaderboard(name: string) { promises.push(fetchLeaderboardPlayer(item)) } const leaderboard = await Promise.all(promises) - return leaderboard + return { + name: name, + unit: getLeaderboardUnit(name) ?? null, + list: leaderboard + } } async function getMemberLeaderboardRequirement(name: string): Promise<number> { @@ -192,14 +216,12 @@ async function getApplicableAttributes(member): Promise<{ [key: string]: number const requirement = await getMemberLeaderboardRequirement(attributeName) if (!requirement || attributeValue > requirement) applicableAttributes[attributeName] = attributeValue - console.log(attributeName) } return applicableAttributes } /** Update the member's leaderboard data on the server if applicable */ export async function updateDatabaseMember(member: CleanMember, profile: CleanFullProfile) { - console.log('updating', member.uuid) if (!client) return // the db client hasn't been initialized // the member's been updated too recently, just return if (recentlyUpdated.get(profile.uuid + member.uuid)) @@ -243,7 +265,6 @@ export async function updateDatabaseMember(member: CleanMember, profile: CleanFu .slice(0, 100) cachedRawLeaderboards.set(attributeName, newRawLeaderboard) } - console.log('updated', member.uuid) } const queue = new Queue({ @@ -265,7 +286,7 @@ async function removeBadMemberLeaderboardAttributes() { // shuffle so if the application is restarting many times itll still be useful for (const leaderboard of shuffle(leaderboards)) { // wait 10 seconds so it doesnt use as much ram - await sleep(100000) + await sleep(10 * 1000) const unsetValue = {} unsetValue[leaderboard] = '' @@ -283,7 +304,26 @@ async function removeBadMemberLeaderboardAttributes() { } } +/** Fetch all the leaderboards, used for caching. Don't call this often! */ +async function fetchAllLeaderboards() { + const leaderboards = await fetchAllMemberLeaderboardAttributes() + // shuffle so if the application is restarting many times itll still be useful + if (debug) console.log('Caching leaderboards!') + for (const leaderboard of shuffle(leaderboards)) { + // wait 2 seconds so it doesnt use as much ram + await sleep(2 * 1000) + + await fetchMemberLeaderboard(leaderboard) + } + if (debug) console.log('Finished caching leaderboards!') +} -connect() - .then(removeBadMemberLeaderboardAttributes) +connect().then(() => { + // when it connects, cache the leaderboards and remove bad members + removeBadMemberLeaderboardAttributes() + // cache leaderboards on startup so its faster later on + fetchAllLeaderboards() + // cache leaderboard players again every hour + setInterval(fetchAllLeaderboards, 4 * 60 * 60 * 1000) +}) diff --git a/src/hypixelCached.ts b/src/hypixelCached.ts index 4063069..5338229 100644 --- a/src/hypixelCached.ts +++ b/src/hypixelCached.ts @@ -29,6 +29,13 @@ const playerCache = new NodeCache({ useClones: true, }) +// cache "basic players" (players without profiles) for 4 hours +const basicPlayerCache = new NodeCache({ + stdTTL: 60 * 60 * 4, + checkperiod: 60 * 10, + useClones: true +}) + const profileCache = new NodeCache({ stdTTL: 30, checkperiod: 10, @@ -130,9 +137,8 @@ export async function usernameFromUser(user: string): Promise<string> { export async function fetchPlayer(user: string): Promise<CleanPlayer> { const playerUuid = await uuidFromUser(user) - if (playerCache.has(playerUuid)) { + if (playerCache.has(playerUuid)) return playerCache.get(playerUuid) - } const cleanPlayer: CleanPlayer = await hypixel.sendCleanApiRequest({ path: 'player', @@ -144,10 +150,25 @@ export async function fetchPlayer(user: string): Promise<CleanPlayer> { // clone in case it gets modified somehow later playerCache.set(playerUuid, cleanPlayer) usernameCache.set(playerUuid, cleanPlayer.username) + + const cleanBasicPlayer = Object.assign({}, cleanPlayer) + delete cleanBasicPlayer.profiles + basicPlayerCache.set(playerUuid, cleanBasicPlayer) return cleanPlayer } +/** Fetch a player without their profiles. This is heavily cached. */ +export async function fetchBasicPlayer(user: string): Promise<CleanPlayer> { + const playerUuid = await uuidFromUser(user) + + if (basicPlayerCache.has(playerUuid)) + return basicPlayerCache.get(playerUuid) + + const player = await fetchPlayer(playerUuid) + delete player.profiles + return player +} export async function fetchSkyblockProfiles(playerUuid: string): Promise<CleanProfile[]> { if (profilesCache.has(playerUuid)) { |