diff options
Diffstat (limited to 'build')
| -rw-r--r-- | build/cleaners/skyblock/member.js | 3 | ||||
| -rw-r--r-- | build/cleaners/skyblock/profile.js | 4 | ||||
| -rw-r--r-- | build/cleaners/skyblock/profiles.js | 3 | ||||
| -rw-r--r-- | build/constants.js | 111 | ||||
| -rw-r--r-- | build/database.js | 115 | ||||
| -rw-r--r-- | build/hypixel.js | 35 | ||||
| -rw-r--r-- | build/hypixelCached.js | 62 | ||||
| -rw-r--r-- | build/index.js | 2 | ||||
| -rw-r--r-- | build/util.js | 2 |
9 files changed, 315 insertions, 22 deletions
diff --git a/build/cleaners/skyblock/member.js b/build/cleaners/skyblock/member.js index 8b922a5..2e07f39 100644 --- a/build/cleaners/skyblock/member.js +++ b/build/cleaners/skyblock/member.js @@ -43,6 +43,7 @@ async function cleanSkyBlockProfileMemberResponseBasic(member, included = null) exports.cleanSkyBlockProfileMemberResponseBasic = cleanSkyBlockProfileMemberResponseBasic; /** Cleans up a member (from skyblock/profile) */ async function cleanSkyBlockProfileMemberResponse(member, included = null) { + var _a; // profiles.members[] const inventoriesIncluded = included == null || included.includes('inventories'); const player = await cached.fetchPlayer(member.uuid); @@ -54,6 +55,8 @@ async function cleanSkyBlockProfileMemberResponse(member, included = null) { rank: player.rank, purse: member.coin_purse, stats: stats_1.cleanProfileStats(member), + // this is used for leaderboards + rawHypixelStats: (_a = member.stats) !== null && _a !== void 0 ? _a : {}, minions: minions_1.cleanMinions(member), fairy_souls: fairysouls_1.cleanFairySouls(member), inventories: inventoriesIncluded ? await inventory_1.cleanInventories(member) : undefined, diff --git a/build/cleaners/skyblock/profile.js b/build/cleaners/skyblock/profile.js index 2b18ff8..fe1161e 100644 --- a/build/cleaners/skyblock/profile.js +++ b/build/cleaners/skyblock/profile.js @@ -25,12 +25,12 @@ exports.cleanSkyblockProfileResponseLighter = cleanSkyblockProfileResponseLighte /** * This function is somewhat costly and shouldn't be called often. Use cleanSkyblockProfileResponseLighter if you don't need all the data */ -async function cleanSkyblockProfileResponse(data, { mainMemberUuid }) { +async function cleanSkyblockProfileResponse(data, options) { const cleanedMembers = []; for (const memberUUID in data.members) { const memberRaw = data.members[memberUUID]; memberRaw.uuid = memberUUID; - const member = await member_1.cleanSkyBlockProfileMemberResponse(memberRaw, ['stats', mainMemberUuid === memberUUID ? 'inventories' : undefined]); + const member = await member_1.cleanSkyBlockProfileMemberResponse(memberRaw, ['stats', (options === null || options === void 0 ? void 0 : options.mainMemberUuid) === memberUUID ? 'inventories' : undefined]); cleanedMembers.push(member); } const memberMinions = []; diff --git a/build/cleaners/skyblock/profiles.js b/build/cleaners/skyblock/profiles.js index b4eb07b..029110a 100644 --- a/build/cleaners/skyblock/profiles.js +++ b/build/cleaners/skyblock/profiles.js @@ -17,7 +17,8 @@ exports.cleanPlayerSkyblockProfiles = cleanPlayerSkyblockProfiles; async function cleanSkyblockProfilesResponse(data) { const cleanedProfiles = []; for (const profile of data !== null && data !== void 0 ? data : []) { - let cleanedProfile = await profile_1.cleanSkyblockProfileResponseLighter(profile); + // let cleanedProfile = await cleanSkyblockProfileResponseLighter(profile) + let cleanedProfile = await profile_1.cleanSkyblockProfileResponse(profile); cleanedProfiles.push(cleanedProfile); } return cleanedProfiles; diff --git a/build/constants.js b/build/constants.js new file mode 100644 index 0000000..23d2b75 --- /dev/null +++ b/build/constants.js @@ -0,0 +1,111 @@ +"use strict"; +/** + * Fetch and edit constants from the skyblock-constants repo + */ +var __importDefault = (this && this.__importDefault) || function (mod) { + return (mod && mod.__esModule) ? mod : { "default": mod }; +}; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.addStats = exports.fetchStats = void 0; +const node_fetch_1 = __importDefault(require("node-fetch")); +const https_1 = require("https"); +const node_cache_1 = __importDefault(require("node-cache")); +const httpsAgent = new https_1.Agent({ + keepAlive: true +}); +const githubApiBase = 'https://api.github.com'; +const owner = 'skyblockstats'; +const repo = 'skyblock-constants'; +/** + * Send a request to the GitHub API + * @param method The HTTP method, for example GET, PUT, POST, etc + * @param route The route to send the request to + * @param headers The extra headers + * @param json The JSON body, only applicable for some types of methods + */ +async function fetchGithubApi(method, route, headers, json) { + return await node_fetch_1.default(githubApiBase + route, { + agent: () => httpsAgent, + body: json ? JSON.stringify(json) : null, + method, + headers: Object.assign({ + 'Authorization': `token ${process.env.github_token}` + }, headers), + }); +} +// cache files for a day +const fileCache = new node_cache_1.default({ + stdTTL: 60 * 60 * 24, + checkperiod: 60, + useClones: false, +}); +/** + * Fetch a file from skyblock-constants + * @param path The file path, for example stats.json + */ +async function fetchFile(path) { + if (fileCache.has(path)) + return fileCache.get(path); + const r = await fetchGithubApi('GET', `/repos/${owner}/${repo}/contents/${path}`, { 'Accept': 'application/vnd.github.v3+json' }); + const data = await r.json(); + return { + path: data.path, + content: Buffer.from(data.content, data.encoding).toString(), + sha: data.sha + }; +} +/** + * Edit a file on skyblock-constants + * @param file The GithubFile you got from fetchFile + * @param message The commit message + * @param newContent The new content in the file + */ +async function editFile(file, message, newContent) { + fileCache.set(file.path, newContent); + await fetchGithubApi('PUT', `/repos/${owner}/${repo}/contents/${file.path}`, { 'Content-Type': 'application/json' }, { + message: message, + content: Buffer.from(newContent).toString('base64'), + sha: file.sha, + branch: 'main' + }); +} +/** Fetch all the known SkyBlock stats as an array of strings */ +async function fetchStats() { + const file = await fetchFile('stats.json'); + try { + return JSON.parse(file.content); + } + catch { + // probably invalid json, return an empty array + return []; + } +} +exports.fetchStats = fetchStats; +/** Add stats to skyblock-constants. This has caching so it's fine to call many times */ +async function addStats(addingStats) { + if (addingStats.length === 0) + return; // no stats provided, just return + const file = await fetchFile('stats.json'); + if (!file.path) + return; + let oldStats; + try { + oldStats = JSON.parse(file.content); + } + catch { + // invalid json, set it as an empty array + oldStats = []; + } + const updatedStats = oldStats + .concat(addingStats) + // remove duplicates + .filter((value, index, array) => array.indexOf(value) === index) + .sort((a, b) => a.localeCompare(b)); + const newStats = updatedStats.filter(value => !oldStats.includes(value)); + // there's not actually any new stats, just return + if (newStats.length === 0) + return; + const commitMessage = newStats.length >= 2 ? `Add ${newStats.length} new stats` : `Add '${newStats[0]}'`; + await editFile(file, commitMessage, JSON.stringify(updatedStats, null, 2)); +} +exports.addStats = addStats; diff --git a/build/database.js b/build/database.js new file mode 100644 index 0000000..31c85ee --- /dev/null +++ b/build/database.js @@ -0,0 +1,115 @@ +"use strict"; +/** + * Store data about members for leaderboards +*/ +var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) { + if (k2 === undefined) k2 = k; + Object.defineProperty(o, k2, { enumerable: true, get: function() { return m[k]; } }); +}) : (function(o, m, k, k2) { + if (k2 === undefined) k2 = k; + o[k2] = m[k]; +})); +var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) { + Object.defineProperty(o, "default", { enumerable: true, value: v }); +}) : function(o, v) { + o["default"] = v; +}); +var __importStar = (this && this.__importStar) || function (mod) { + if (mod && mod.__esModule) return mod; + var result = {}; + if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k); + __setModuleDefault(result, mod); + return result; +}; +var __importDefault = (this && this.__importDefault) || function (mod) { + return (mod && mod.__esModule) ? mod : { "default": mod }; +}; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.updateDatabaseMember = void 0; +const constants = __importStar(require("./constants")); +const mongodb_1 = require("mongodb"); +const node_cache_1 = __importDefault(require("node-cache")); +// don't update the user for 3 minutes +const recentlyUpdated = new node_cache_1.default({ + stdTTL: 60 * 3, + checkperiod: 60, + useClones: false, +}); +const cachedLeaderboards = new Map(); +let client; +let database; +let memberLeaderboardsCollection; +async function connect() { + if (!process.env.db_uri) + return console.warn('Warning: db_uri was not found in .env. Features that utilize the database such as leaderboards won\'t work.'); + if (!process.env.db_name) + return console.warn('Warning: db_name was not found in .env. Features that utilize the database such as leaderboards won\'t work.'); + client = await mongodb_1.MongoClient.connect(process.env.db_uri, { useNewUrlParser: true, useUnifiedTopology: true }); + database = client.db(process.env.db_name); + memberLeaderboardsCollection = database.collection('member-leaderboards'); +} +function getMemberCollectionAttributes(member) { + const collectionAttributes = {}; + for (const collection of member.collections) { + const collectionLeaderboardName = `collection_${collection.name}`; + collectionAttributes[collectionLeaderboardName] = collection.xp; + } + return collectionAttributes; +} +function getMemberLeaderboardAttributes(member) { + // if you want to add a new leaderboard for member attributes, add it here + return { + // we use the raw stat names rather than the clean stats in case hypixel adds a new stat and it takes a while for us to clean it + ...member.rawHypixelStats, + // collection leaderboards + ...getMemberCollectionAttributes(member), + fairy_souls: member.fairy_souls.total, + first_join: member.first_join, + purse: member.purse, + visited_zones: member.visited_zones.length, + }; +} +async function fetchMemberLeaderboard(name) { + if (cachedLeaderboards.has(name)) + return cachedLeaderboards.get(name); + // typescript forces us to make a new variable and set it this way because it gives an error otherwise + const query = {}; + query[`stats.${name}`] = { '$exists': true }; + const sortQuery = {}; + sortQuery[`stats.${name}`] = 1; + const leaderboard = await memberLeaderboardsCollection.find(query).sort(sortQuery).toArray(); + cachedLeaderboards.set(name, leaderboard); + return leaderboard; +} +async function getLeaderboardRequirement(name) { + const leaderboard = await fetchMemberLeaderboard(name); + // if there's more than 100 items, return the 100th. if there's less, return null + if (leaderboard.length > 100) + return leaderboard[100].stats[name]; + else + return null; +} +/** Update the member's leaderboard data on the server if applicable */ +async function updateDatabaseMember(member) { + if (!client) + return; // the db client hasn't been initialized + // the member's been updated too recently, just return + if (recentlyUpdated.get(member.uuid)) + return; + // store the member in recentlyUpdated so it cant update for 3 more minutes + recentlyUpdated.set(member.uuid, true); + await constants.addStats(Object.keys(member.rawHypixelStats)); + const leaderboardAttributes = getMemberLeaderboardAttributes(member); + await memberLeaderboardsCollection.updateOne({ + uuid: member.uuid + }, { + '$set': { + 'stats': leaderboardAttributes, + 'last_updated': new Date() + } + }, { + upsert: true + }); +} +exports.updateDatabaseMember = updateDatabaseMember; +connect(); diff --git a/build/hypixel.js b/build/hypixel.js index 6e6b8b6..97e24b0 100644 --- a/build/hypixel.js +++ b/build/hypixel.js @@ -22,17 +22,19 @@ var __importStar = (this && this.__importStar) || function (mod) { return result; }; Object.defineProperty(exports, "__esModule", { value: true }); -exports.fetchMemberProfile = exports.fetchUser = exports.sendCleanApiRequest = exports.maxMinion = exports.saveInterval = void 0; +exports.fetchMemberProfilesUncached = exports.fetchMemberProfileUncached = exports.fetchMemberProfile = exports.fetchUser = exports.sendCleanApiRequest = exports.maxMinion = exports.saveInterval = void 0; const player_1 = require("./cleaners/player"); const hypixelApi_1 = require("./hypixelApi"); const cached = __importStar(require("./hypixelCached")); const profile_1 = require("./cleaners/skyblock/profile"); const profiles_1 = require("./cleaners/skyblock/profiles"); const _1 = require("."); +const database_1 = require("./database"); // the interval at which the "last_save" parameter updates in the hypixel api, this is 3 minutes exports.saveInterval = 60 * 3 * 1000; // the highest level a minion can be exports.maxMinion = 11; +/** Sends an API request to Hypixel and cleans it up. */ async function sendCleanApiRequest({ path, args }, included, options) { const key = await hypixelApi_1.chooseApiKey(); const rawResponse = await hypixelApi_1.sendApiRequest({ path, key, args }); @@ -143,3 +145,34 @@ async function fetchMemberProfile(user, profile) { }; } exports.fetchMemberProfile = fetchMemberProfile; +/** + * Fetches the Hypixel API to get a CleanFullProfile. This doesn't do any caching and you should use hypixelCached.fetchProfile instead + * @param playerUuid The UUID of the Minecraft player + * @param profileUuid The UUID of the Hypixel SkyBlock profile + */ +async function fetchMemberProfileUncached(playerUuid, profileUuid) { + const profile = await sendCleanApiRequest({ + path: 'skyblock/profile', + args: { profile: profileUuid } + }, null, { mainMemberUuid: playerUuid }); + for (const member of profile.members) + database_1.updateDatabaseMember(member); + return profile; +} +exports.fetchMemberProfileUncached = fetchMemberProfileUncached; +async function fetchMemberProfilesUncached(playerUuid) { + const profiles = await sendCleanApiRequest({ + path: 'skyblock/profiles', + args: { + uuid: playerUuid + } + }, null, { + // only the inventories for the main player are generated, this is for optimization purposes + mainMemberUuid: playerUuid + }); + for (const profile of profiles) + for (const member of profile.members) + database_1.updateDatabaseMember(member); + return profiles; +} +exports.fetchMemberProfilesUncached = fetchMemberProfilesUncached; diff --git a/build/hypixelCached.js b/build/hypixelCached.js index 2769a42..5a970f8 100644 --- a/build/hypixelCached.js +++ b/build/hypixelCached.js @@ -62,26 +62,53 @@ const profileNameCache = new node_cache_1.default({ checkperiod: 60, useClones: false, }); +function waitForSet(cache, key, value) { + return new Promise((resolve, reject) => { + const listener = (setKey, setValue) => { + if (setKey === key || (value && setValue === value)) { + cache.removeListener('set', listener); + return resolve({ key, value }); + } + }; + cache.on('set', listener); + }); +} /** * Fetch the uuid from a user * @param user A user can be either a uuid or a username */ async function uuidFromUser(user) { - if (usernameCache.has(util_1.undashUuid(user))) + if (usernameCache.has(util_1.undashUuid(user))) { // check if the uuid is a key - return util_1.undashUuid(user); + const username = usernameCache.get(util_1.undashUuid(user)); + // if it has .then, then that means its a waitForSet promise + if (username.then) { + console.log('pog, prevented double request'); + return (await username()).key; + } + else + return util_1.undashUuid(user); + } // check if the username is a value const uuidToUsername = usernameCache.mget(usernameCache.keys()); for (const [uuid, username] of Object.entries(uuidToUsername)) { if (user.toLowerCase() === username.toLowerCase()) return uuid; } + if (_1.debug) + console.log('Cache miss: uuidFromUser', user); + // set it as waitForSet (a promise) in case uuidFromUser gets called while its fetching mojang + console.log('setting', util_1.undashUuid(user)); + usernameCache.set(util_1.undashUuid(user), waitForSet(usernameCache, user, user)); + console.log(util_1.undashUuid(user), usernameCache.has(util_1.undashUuid(user))); // not cached, actually fetch mojang api now let { uuid, username } = await mojang.mojangDataFromUser(user); if (!uuid) return; // remove dashes from the uuid so its more normal uuid = util_1.undashUuid(uuid); + if (user !== uuid) + usernameCache.del(user); usernameCache.set(uuid, username); return uuid; } @@ -96,6 +123,8 @@ async function usernameFromUser(user) { console.log('Cache hit! usernameFromUser', user); return usernameCache.get(util_1.undashUuid(user)); } + if (_1.debug) + console.log('Cache miss: usernameFromUser', user); let { uuid, username } = await mojang.mojangDataFromUser(user); uuid = util_1.undashUuid(uuid); usernameCache.set(uuid, username); @@ -106,9 +135,11 @@ async function fetchPlayer(user) { const playerUuid = await uuidFromUser(user); if (playerCache.has(playerUuid)) { if (_1.debug) - console.log('Cache hit! fetchPlayer', playerUuid); + console.log('Cache hit! fetchPlayer', user); return playerCache.get(playerUuid); } + if (_1.debug) + console.log('Cache miss: uuidFromUser', user); const cleanPlayer = await hypixel.sendCleanApiRequest({ path: 'player', args: { uuid: playerUuid } @@ -126,15 +157,9 @@ async function fetchSkyblockProfiles(playerUuid) { console.log('Cache hit! fetchSkyblockProfiles', playerUuid); return profilesCache.get(playerUuid); } - const profiles = await hypixel.sendCleanApiRequest({ - path: 'skyblock/profiles', - args: { - uuid: playerUuid - } - }, null, { - // only the inventories for the main player are generated, this is for optimization purposes - mainMemberUuid: playerUuid - }); + if (_1.debug) + console.log('Cache miss: fetchSkyblockProfiles', playerUuid); + const profiles = await hypixel.fetchMemberProfilesUncached(playerUuid); const basicProfiles = []; // create the basicProfiles array for (const profile of profiles) { @@ -166,6 +191,8 @@ async function fetchBasicProfiles(user) { console.log('Cache hit! fetchBasicProfiles', playerUuid); return basicProfilesCache.get(playerUuid); } + if (_1.debug) + console.log('Cache miss: fetchBasicProfiles', user); const player = await fetchPlayer(playerUuid); const profiles = player.profiles; basicProfilesCache.set(playerUuid, profiles); @@ -186,6 +213,8 @@ async function fetchProfileUuid(user, profile) { console.log('no profile provided?', user, profile); return null; } + if (_1.debug) + console.log('Cache miss: fetchProfileUuid', user); const profiles = await fetchBasicProfiles(user); const profileUuid = util_1.undashUuid(profile); for (const p of profiles) { @@ -210,11 +239,10 @@ async function fetchProfile(user, profile) { console.log('Cache hit! fetchProfile', profileUuid); return profileCache.get(profileUuid); } + if (_1.debug) + console.log('Cache miss: fetchProfile', user, profile); const profileName = await fetchProfileName(user, profile); - const cleanProfile = await hypixel.sendCleanApiRequest({ - path: 'skyblock/profile', - args: { profile: profileUuid } - }, null, { mainMemberUuid: playerUuid }); + const cleanProfile = await hypixel.fetchMemberProfileUncached(playerUuid, profileUuid); // we know the name from fetchProfileName, so set it here cleanProfile.name = profileName; profileCache.set(profileUuid, cleanProfile); @@ -236,6 +264,8 @@ async function fetchProfileName(user, profile) { console.log('Cache hit! fetchProfileName', profileUuid); return profileNameCache.get(`${playerUuid}.${profileUuid}`); } + if (_1.debug) + console.log('Cache miss: fetchProfileName', user, profile); const basicProfiles = await fetchBasicProfiles(playerUuid); let profileName; for (const basicProfile of basicProfiles) diff --git a/build/index.js b/build/index.js index b089351..3174e69 100644 --- a/build/index.js +++ b/build/index.js @@ -7,7 +7,7 @@ exports.debug = void 0; const hypixel_1 = require("./hypixel"); const express_1 = __importDefault(require("express")); const app = express_1.default(); -exports.debug = false; +exports.debug = true; app.use((req, res, next) => { if (process.env.key && req.headers.key !== process.env.key) // if a key is set in process.env and the header doesn't match return an error diff --git a/build/util.js b/build/util.js index 3350acf..3b21db4 100644 --- a/build/util.js +++ b/build/util.js @@ -5,7 +5,7 @@ Object.defineProperty(exports, "__esModule", { value: true }); exports.colorCodeFromName = exports.minecraftColorCodes = exports.shuffle = exports.jsonToQuery = exports.queryToJson = exports.undashUuid = void 0; function undashUuid(uuid) { - return uuid.replace(/-/g, ''); + return uuid.replace(/-/g, '').toLowerCase(); } exports.undashUuid = undashUuid; function queryToJson(queryString) { |