Initial Modpack Upload

This commit is contained in:
Rbanh committed 2023-08-14 21:45:09 -04:00
1 parent 268b0650aa
commit 3d8b956ee4
1429 files changed
+820128

No files matched your search

+759
View File
@@ -0,0 +1,759 @@
"use strict";
var __create = Object.create;
var __defProp = Object.defineProperty;
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
var __getOwnPropNames = Object.getOwnPropertyNames;
var __getProtoOf = Object.getPrototypeOf;
var __hasOwnProp = Object.prototype.hasOwnProperty;
var __export = (target, all) => {
for (var name in all)
__defProp(target, name, { get: all[name], enumerable: true });
};
var __copyProps = (to, from, except, desc) => {
if (from && typeof from === "object" || typeof from === "function") {
for (let key of __getOwnPropNames(from))
if (!__hasOwnProp.call(to, key) && key !== except)
__defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
}
return to;
};
var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
// If the importer is in node compatibility mode or this is not an ESM
// file that has been converted to a CommonJS file using a Babel-
// compatible transform (i.e. "__esModule" has not been set), then set
// "default" to the CommonJS "module.exports" for node compatibility.
isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
mod
));
var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
var converter_exports = {};
__export(converter_exports, {
ModlogConverter: () => ModlogConverter,
ModlogConverterSQLite: () => ModlogConverterSQLite,
ModlogConverterTest: () => ModlogConverterTest,
ModlogConverterTxt: () => ModlogConverterTxt,
modernizeLog: () => modernizeLog,
parseBrackets: () => parseBrackets,
parseModlog: () => parseModlog,
rawifyLog: () => rawifyLog
});
module.exports = __toCommonJS(converter_exports);
var import_lib = require("../../lib");
var import_ip_tools = require("../../server/ip-tools");
if (!global.Config) {
let hasSQLite = true;
try {
require.resolve("better-sqlite3");
} catch {
console.warn(`Warning: the modlog conversion script is running without a SQLite library.`);
hasSQLite = false;
}
global.Config = {
nofswriting: false,
usesqlitemodlog: hasSQLite,
usesqlite: hasSQLite
};
}
const Database = Config.usesqlite ? require("better-sqlite3") : null;
const { Modlog } = require("../../server/modlog");
const ENTRIES_TO_BUFFER = 7500;
const ALTS_REGEX = /\(.*?'s (lock|mut|bann|blacklist)ed alts: (.*)\)/;
const AUTOCONFIRMED_REGEX = /\(.*?'s ac account: (.*)\)/;
const IP_ONLY_ACTIONS = /* @__PURE__ */ new Set([
"SHAREDIP",
"UNSHAREDIP",
"UNLOCKIP",
"UNLOCKRANGE",
"RANGEBAN",
"RANGELOCK"
]);
function parseBrackets(line, openingBracket, greedy) {
const brackets = {
"(": ")",
"[": "]"
};
const bracketOpenIndex = line.indexOf(openingBracket);
const bracketCloseIndex = greedy ? line.lastIndexOf(brackets[openingBracket]) : line.indexOf(brackets[openingBracket]);
if (bracketCloseIndex < 0 || bracketOpenIndex < 0)
return "";
return line.slice(bracketOpenIndex + 1, bracketCloseIndex);
}
function toID(text) {
return (text && typeof text === "string" ? text : "").toLowerCase().replace(/[^a-z0-9]+/g, "");
}
function modernizeLog(line, nextLine) {
const prefix = line.match(/\[.+?\] \(.+?\) /i)?.[0];
if (!prefix)
return;
if (ALTS_REGEX.test(line) || AUTOCONFIRMED_REGEX.test(line))
return;
line = line.replace(prefix, "");
if (line.startsWith("("))
line = line.replace(/\([a-z0-9-]*\) /, "");
if (line.startsWith("(") && line.endsWith(")")) {
line = line.slice(1, -1);
}
const getAlts = () => {
let alts = "";
nextLine?.replace(ALTS_REGEX, (_a, _b, rawAlts) => {
if (rawAlts)
alts = `alts: [${rawAlts.split(",").map(toID).join("], [")}] `;
return "";
});
return alts;
};
const getAutoconfirmed = () => {
let autoconfirmed = "";
nextLine?.replace(AUTOCONFIRMED_REGEX, (_a, rawAutoconfirmed) => {
if (rawAutoconfirmed)
autoconfirmed = `ac: [${toID(rawAutoconfirmed)}] `;
return "";
});
return autoconfirmed;
};
if (line.startsWith("SCAV ")) {
line = line.replace(/: (\[room: .*?\]) by (.*)/, (match, roominfo, rest) => `: by ${rest} ${roominfo}`);
}
line = line.replace(
/(GIVEAWAY WIN|GTS FINISHED): ([A-Za-z0-9].*?)(won|has finished)/,
(match, action, user) => `${action}: [${toID(user)}]:`
);
if (line.includes(":")) {
const possibleModernAction = line.slice(0, line.indexOf(":")).trim();
if (possibleModernAction === possibleModernAction.toUpperCase()) {
if (possibleModernAction.includes("[")) {
const [drop, ...keep] = line.split("[");
process.stderr.write(`Ignoring malformed line: ${drop}
`);
return modernizeLog(keep.join(""));
}
if (/\(.+\) by [a-z0-9]{1,19}$/.test(line) && !["OLD MODLOG", "NOTE"].includes(possibleModernAction)) {
const reason = parseBrackets(line, "(", true);
return `${prefix}${line.replace(` (${reason})`, "")}: ${reason}`;
}
return `${prefix}${line}`;
}
}
if (/\[(the|a)poll\] was (started|ended) by/.test(line)) {
const actionTaker = toID(line.slice(line.indexOf(" by ") + " by ".length));
const isEnding = line.includes("was ended by");
return `${prefix}POLL${isEnding ? " END" : ""}: by ${actionTaker}`;
}
if (/User (.*?) won the game of (.*?) mode trivia/.test(line)) {
return `${prefix}TRIVIAGAME: by unknown: ${line}`;
}
const modernizerTransformations = {
"notes: ": (log) => {
const [actionTaker, ...rest] = line.split(" notes: ");
return `NOTE: by ${toID(actionTaker)}: ${rest.join("")}`;
},
" declared": (log) => {
let newAction = "DECLARE";
let oldAction = " declared";
if (log.includes(" globally declared")) {
oldAction = " globally declared";
newAction = "GLOBALDECLARE";
}
if (log.includes("(chat level)")) {
oldAction += " (chat level)";
newAction = `CHATDECLARE`;
}
const actionTakerName = toID(log.slice(0, log.lastIndexOf(oldAction)));
log = log.slice(actionTakerName.length);
log = log.slice(oldAction.length);
log = log.replace(/^\s?:/, "").trim();
return `${newAction}: by ${actionTakerName}: ${log}`;
},
"changed the roomdesc to: ": (log) => {
const actionTaker = parseBrackets(log, "[");
log = log.slice(actionTaker.length + 3);
log = log.slice("changed the roomdesc to: ".length + 1, -2);
return `ROOMDESC: by ${actionTaker}: to "${log}"`;
},
'roomevent titled "': (log) => {
let action;
if (log.includes(' added a roomevent titled "')) {
action = "added a";
} else if (log.includes(' removed a roomevent titled "')) {
action = "removed a";
} else {
action = "edited the";
}
const actionTakerName = log.slice(0, log.lastIndexOf(` ${action} roomevent titled "`));
log = log.slice(actionTakerName.length + 1);
const eventName = log.slice(` ${action} roomevent titled `.length, -2);
return `ROOMEVENT: by ${toID(actionTakerName)}: ${action.split(" ")[0]} "${eventName}"`;
},
"set modchat to ": (log) => {
const actionTaker = parseBrackets(log, "[");
log = log.slice(actionTaker.length + 3);
log = log.slice("set modchat to ".length);
return `MODCHAT: by ${actionTaker}: to ${log}`;
},
"set modjoin to ": (log) => {
const actionTakerName = log.slice(0, log.lastIndexOf(" set"));
log = log.slice(actionTakerName.length + 1);
log = log.slice("set modjoin to ".length);
const rank = log.startsWith("sync") ? "sync" : log.replace(".", "");
return `MODJOIN${rank === "sync" ? " SYNC" : ""}: by ${toID(actionTakerName)}${rank !== "sync" ? `: ${rank}` : ``}`;
},
"turned off modjoin": (log) => {
const actionTakerName = log.slice(0, log.lastIndexOf(" turned off modjoin"));
return `MODJOIN: by ${toID(actionTakerName)}: OFF`;
},
"changed the roomintro": (log) => {
const isDeletion = /deleted the (staff|room)intro/.test(log);
const isRoomintro = log.includes("roomintro");
const actionTaker = toID(log.slice(0, log.indexOf(isDeletion ? "deleted" : "changed")));
return `${isDeletion ? "DELETE" : ""}${isRoomintro ? "ROOM" : "STAFF"}INTRO: by ${actionTaker}`;
},
"deleted the roomintro": (log) => modernizerTransformations["changed the roomintro"](log),
"changed the staffintro": (log) => modernizerTransformations["changed the roomintro"](log),
"deleted the staffintro": (log) => modernizerTransformations["changed the roomintro"](log),
"created a tournament in": (log) => {
const actionTaker = parseBrackets(log, "[");
log = log.slice(actionTaker.length + 3);
log = log.slice(24, -8);
return `TOUR CREATE: by ${actionTaker}: ${log}`;
},
"was disqualified from the tournament by": (log) => {
const disqualified = parseBrackets(log, "[");
log = log.slice(disqualified.length + 3);
log = log.slice("was disqualified from the tournament by".length);
return `TOUR DQ: [${toID(disqualified)}] by ${toID(log)}`;
},
"The tournament auto disqualify timeout was set to": (log) => {
const byIndex = log.indexOf(" by ");
const actionTaker = log.slice(byIndex + " by ".length);
const length = log.slice("The tournament auto disqualify timeout was set to".length, byIndex);
return `TOUR AUTODQ: by ${toID(actionTaker)}: ${length.trim()}`;
},
" was blacklisted from ": (log) => {
const isName = log.includes(" was nameblacklisted from ");
const banned = toID(log.slice(0, log.indexOf(` was ${isName ? "name" : ""}blacklisted from `)));
log = log.slice(log.indexOf(" by ") + " by ".length);
let reason, ip;
if (/\(.*\)/.test(log)) {
reason = parseBrackets(log, "(");
if (/\[.*\]/.test(log))
ip = parseBrackets(log, "[");
log = log.slice(0, log.indexOf("("));
}
const actionTaker = toID(log);
return `${isName ? "NAME" : ""}BLACKLIST: [${banned}] ${getAutoconfirmed()}${getAlts()}${ip ? `[${ip}] ` : ``}by ${actionTaker}${reason ? `: ${reason}` : ``}`;
},
" was nameblacklisted from ": (log) => modernizerTransformations[" was blacklisted from "](log),
" was banned from room ": (log) => {
const banned = toID(log.slice(0, log.indexOf(" was banned from room ")));
log = log.slice(log.indexOf(" by ") + " by ".length);
let reason, ip;
if (/\(.*\)/.test(log)) {
reason = parseBrackets(log, "(");
if (/\[.*\]/.test(log))
ip = parseBrackets(log, "[");
log = log.slice(0, log.indexOf("("));
}
const actionTaker = toID(log);
return `ROOMBAN: [${banned}] ${getAutoconfirmed()}${getAlts()}${ip ? `[${ip}] ` : ``}by ${actionTaker}${reason ? `: ${reason}` : ``}`;
},
" was muted by ": (log) => {
let muted = "";
let isHour = false;
[muted, log] = log.split(" was muted by ");
muted = toID(muted);
let reason, ip;
if (/\(.*\)/.test(log)) {
reason = parseBrackets(log, "(");
if (/\[.*\]/.test(log))
ip = parseBrackets(log, "[");
log = log.slice(0, log.indexOf("("));
}
let actionTaker = toID(log);
if (actionTaker.endsWith("for1hour")) {
isHour = true;
actionTaker = actionTaker.replace(/^(.*)(for1hour)$/, (match, staff) => staff);
}
return `${isHour ? "HOUR" : ""}MUTE: [${muted}] ${getAutoconfirmed()}${getAlts()}${ip ? `[${ip}] ` : ``}by ${actionTaker}${reason ? `: ${reason}` : ``}`;
},
" was locked from talking ": (log) => {
const isWeek = log.includes(" was locked from talking for a week ");
const locked = toID(log.slice(0, log.indexOf(" was locked from talking ")));
log = log.slice(log.indexOf(" by ") + " by ".length);
let reason, ip;
if (/\(.*\)/.test(log)) {
reason = parseBrackets(log, "(");
if (/\[.*\]/.test(log))
ip = parseBrackets(log, "[");
log = log.slice(0, log.indexOf("("));
}
const actionTaker = toID(log);
return `${isWeek ? "WEEK" : ""}LOCK: [${locked}] ${getAutoconfirmed()}${getAlts()}${ip ? `[${ip}] ` : ``}by ${actionTaker}${reason ? `: ${reason}` : ``}`;
},
" was banned ": (log) => {
if (log.includes(" was banned from room "))
return modernizerTransformations[" was banned from room "](log);
const banned = toID(log.slice(0, log.indexOf(" was banned ")));
log = log.slice(log.indexOf(" by ") + " by ".length);
let reason, ip;
if (/\(.*\)/.test(log)) {
reason = parseBrackets(log, "(");
if (/\[.*\]/.test(log))
ip = parseBrackets(log, "[");
log = log.slice(0, log.indexOf("("));
}
const actionTaker = toID(log);
return `BAN: [${banned}] ${getAutoconfirmed()}${getAlts()}${ip ? `[${ip}] ` : ``}by ${actionTaker}${reason ? `: ${reason}` : ``}`;
},
"was promoted to ": (log) => {
const isDemotion = log.includes("was demoted to ");
const userid = toID(log.split(" was ")[0]);
if (!userid) {
throw new Error(`Ignoring malformed line: ${prefix}${log}`);
}
log = log.slice(userid.length + 3);
log = log.slice(`was ${isDemotion ? "demoted" : "promoted"} to `.length);
let rank = log.slice(0, log.indexOf(" by")).replace(/ /, "").toUpperCase();
log = log.slice(`${rank} by `.length);
if (!rank.startsWith("ROOM"))
rank = `GLOBAL ${rank}`;
const actionTaker = parseBrackets(log, "[");
return `${rank}: [${userid}] by ${actionTaker}${isDemotion ? ": (demote)" : ""}`;
},
"was demoted to ": (log) => modernizerTransformations["was promoted to "](log),
"was appointed Room Owner by ": (log) => {
const userid = parseBrackets(log, "[");
log = log.slice(userid.length + 3);
log = log.slice("was appointed Room Owner by ".length);
const actionTaker = parseBrackets(log, "[");
return `ROOMOWNER: [${userid}] by ${actionTaker}`;
},
" claimed this ticket": (log) => {
const actions = {
" claimed this ticket": "TICKETCLAIM",
" closed this ticket": "TICKETCLOSE",
" deleted this ticket": "TICKETDELETE"
};
for (const oldAction in actions) {
if (log.includes(oldAction)) {
const actionTaker = toID(log.slice(0, log.indexOf(oldAction)));
return `${actions[oldAction]}: by ${actionTaker}`;
}
}
return log;
},
"This ticket is now claimed by ": (log) => {
const claimer = toID(log.slice(log.indexOf(" by ") + " by ".length));
return `TICKETCLAIM: by ${claimer}`;
},
" is no longer interested in this ticket": (log) => {
const abandoner = toID(log.slice(0, log.indexOf(" is no longer interested in this ticket")));
return `TICKETABANDON: by ${abandoner}`;
},
" opened a new ticket": (log) => {
const opener = toID(log.slice(0, log.indexOf(" opened a new ticket")));
const problem = log.slice(log.indexOf(" Issue: ") + " Issue: ".length).trim();
return `TICKETOPEN: by ${opener}: ${problem}`;
},
" closed this ticket": (log) => modernizerTransformations[" claimed this ticket"](log),
" deleted this ticket": (log) => modernizerTransformations[" claimed this ticket"](log),
"This ticket is no longer claimed": () => "TICKETUNCLAIM",
" has been caught attempting a hunt with ": (log) => {
const index = log.indexOf(" has been caught attempting a hunt with ");
const user = toID(log.slice(0, index));
log = log.slice(index + " has been caught attempting a hunt with ".length);
log = log.replace(". The user has also", "; has also").replace(".", "");
return `SCAV CHEATER: [${user}]: caught attempting a hunt with ${log}`;
},
"made this room hidden": (log) => {
const user = toID(log.slice(0, log.indexOf(" made this room hidden")));
return `HIDDENROOM: by ${user}`;
},
"The tournament auto start timer was set to ": (log) => {
log = log.slice("The tournament auto start timer was set to".length);
const [length, setter] = log.split(" by ").map(toID);
return `TOUR AUTOSTART: by ${setter}: ${length}`;
},
"The tournament auto disqualify timer was set to ": (log) => {
log = log.slice("The tournament auto disqualify timer was set to".length);
const [length, setter] = log.split(" by ").map(toID);
return `TOUR AUTODQ: by ${setter}: ${length}`;
},
" set the tournament's banlist to ": (log) => {
const [setter, banlist] = log.split(` set the tournament's banlist to `);
return `TOUR BANLIST: by ${toID(setter)}: ${banlist.slice(0, -1)}`;
},
" set the tournament's custom rules to": (log) => {
const [setter, rules] = log.split(` set the tournament's custom rules to `);
return `TOUR RULES: by ${toID(setter)}: ${rules.slice(0, -1)}`;
},
"[agameofhangman] was started by ": (log) => `HANGMAN: by ${toID(log.slice("[agameofhangman] was started by ".length))}`,
"[agameofunowas] created by ": (log) => `UNO CREATE: by ${toID(log.slice("[agameofunowas] created by ".length))}`,
"[thetournament] was set to autostart": (log) => {
const [, user] = log.split(" by ");
return `TOUR AUTOSTART: by ${toID(user)}: when playercap is reached`;
},
"[thetournament] was set to allow scouting": (log) => {
const [, user] = log.split(" by ");
return `TOUR SCOUT: by ${toID(user)}: allow`;
},
"[thetournament] was set to disallow scouting": (log) => {
const [, user] = log.split(" by ");
return `TOUR SCOUT: by ${toID(user)}: disallow`;
}
};
for (const oldAction in modernizerTransformations) {
if (line.includes(oldAction)) {
try {
return prefix + modernizerTransformations[oldAction](line);
} catch (err) {
if (Config.nofswriting)
throw err;
process.stderr.write(`${err.message}
`);
}
}
}
return `${prefix}${line}`;
}
function parseModlog(raw, nextLine, isGlobal = false) {
let line = modernizeLog(raw);
if (!line)
return;
const timestamp = parseBrackets(line, "[");
line = line.slice(timestamp.length + 3);
const [roomID, ...bonus] = parseBrackets(line, "(").split(" ");
const log = {
action: "NULL",
roomID,
visualRoomID: "",
userid: null,
autoconfirmedID: null,
alts: [],
ip: null,
isGlobal,
loggedBy: null,
note: "",
time: Math.floor(new Date(timestamp).getTime()) || 0
};
if (bonus.length)
log.visualRoomID = `${log.roomID} ${bonus.join(" ")}`;
line = line.slice((log.visualRoomID || log.roomID).length + 3);
const actionColonIndex = line.indexOf(":");
const action = line.slice(0, actionColonIndex);
if (action !== action.toUpperCase()) {
log.action = "OLD MODLOG";
log.loggedBy = "unknown";
log.note = line.trim();
return log;
} else {
log.action = action;
if (log.action === "OLD MODLOG") {
log.loggedBy = "unknown";
log.note = line.slice(line.indexOf("by unknown: ") + "by unknown :".length).trim();
return log;
}
line = line.slice(actionColonIndex + 2);
}
if (line[0] === "[") {
if (!IP_ONLY_ACTIONS.has(log.action)) {
const userid = toID(parseBrackets(line, "["));
log.userid = userid;
line = line.slice(userid.length + 3).trim();
if (line.startsWith("ac:")) {
line = line.slice(3).trim();
const ac = parseBrackets(line, "[");
log.autoconfirmedID = toID(ac);
line = line.slice(ac.length + 3).trim();
}
if (line.startsWith("alts:")) {
line = line.slice(5).trim();
const alts = /* @__PURE__ */ new Set();
let alt = parseBrackets(line, "[");
do {
if (alt.includes(", ")) {
for (const trueAlt of alt.split(", ")) {
alts.add(toID(trueAlt));
}
line = line.slice(line.indexOf(`[${alt}],`) + `[${alt}],`.length).trim();
if (!line.startsWith("["))
line = `[${line}`;
} else {
if (import_ip_tools.IPTools.ipRegex.test(alt))
break;
alts.add(toID(alt));
line = line.slice(line.indexOf(`[${alt}],`) + `[${alt}],`.length).trim();
if (alt.includes("[") && !line.startsWith("["))
line = `[${line}`;
}
alt = parseBrackets(line, "[");
} while (alt);
log.alts = [...alts];
}
}
if (line[0] === "[") {
log.ip = parseBrackets(line, "[");
line = line.slice(log.ip.length + 3).trim();
}
}
let regex = /\bby .*:/;
let actionTakerIndex = regex.exec(line)?.index;
if (actionTakerIndex === void 0) {
actionTakerIndex = line.indexOf("by ");
regex = /\bby .*/;
}
if (actionTakerIndex !== -1) {
const colonIndex = line.indexOf(": ");
const actionTaker = line.slice(actionTakerIndex + 3, colonIndex > actionTakerIndex ? colonIndex : void 0);
if (toID(actionTaker).length < 19) {
log.loggedBy = toID(actionTaker) || null;
if (colonIndex > actionTakerIndex)
line = line.slice(colonIndex);
line = line.replace(regex, " ");
}
}
if (line)
log.note = line.replace(/^\s?:\s?/, "").trim();
return log;
}
function rawifyLog(log) {
let result = `[${new Date(log.time || Date.now()).toJSON()}] (${(log.visualRoomID || log.roomID || "global").replace(/^global-/, "")}) ${log.action}`;
if (log.userid)
result += `: [${log.userid}]`;
if (log.autoconfirmedID)
result += ` ac: [${log.autoconfirmedID}]`;
if (log.alts.length)
result += ` alts: [${log.alts.join("], [")}]`;
if (log.ip) {
if (!log.userid)
result += `:`;
result += ` [${log.ip}]`;
}
if (log.loggedBy)
result += `${result.endsWith("]") ? "" : ":"} by ${log.loggedBy}`;
if (log.note)
result += `: ${log.note}`;
return result + `
`;
}
class ModlogConverterSQLite {
constructor(databaseFile, textLogDir, isTesting, newestAllowedTimestamp) {
this.isTesting = null;
this.databaseFile = databaseFile;
this.textLogDir = textLogDir;
if (isTesting || Config.nofswriting) {
this.isTesting = { files: /* @__PURE__ */ new Map(), db: isTesting || new Database(":memory:") };
}
this.newestAllowedTimestamp = newestAllowedTimestamp;
}
async toTxt() {
const database = this.isTesting?.db || new Database(this.databaseFile, { fileMustExist: true });
const roomids = database.prepare("SELECT DISTINCT roomid FROM modlog").all();
const globalEntries = [];
for (const { roomid } of roomids) {
if (!Config.nofswriting)
console.log(`Reading ${roomid}...`);
const results = database.prepare(
`SELECT *, (SELECT group_concat(userid, ',') FROM alts WHERE alts.modlog_id = modlog.modlog_id) as alts FROM modlog WHERE roomid = ? ORDER BY timestamp ASC`
).all(roomid);
const trueRoomID = roomid.replace(/^global-/, "");
let entriesLogged = 0;
let entries = [];
const insertEntries = async () => {
if (roomid === "global")
return;
entriesLogged += entries.length;
if (!Config.nofswriting && (entriesLogged % ENTRIES_TO_BUFFER === 0 || entriesLogged < ENTRIES_TO_BUFFER)) {
process.stdout.clearLine(0);
process.stdout.cursorTo(0);
process.stdout.write(`Wrote ${entriesLogged} entries from '${trueRoomID}'`);
}
await this.writeFile(`${this.textLogDir}/modlog_${trueRoomID}.txt`, entries.join(""));
entries = [];
};
for (const result of results) {
if (this.newestAllowedTimestamp && result.timestamp > this.newestAllowedTimestamp)
break;
const entry = {
action: result.action,
roomID: result.roomid?.replace(/^global-/, ""),
visualRoomID: result.visual_roomid,
userid: result.userid,
autoconfirmedID: result.autoconfirmed_userid,
alts: result.alts?.split(","),
ip: result.ip,
isGlobal: result.roomid?.startsWith("global-") || result.roomid === "global" || result.is_global,
loggedBy: result.action_taker_userid,
note: result.note,
time: result.timestamp
};
const rawLog = rawifyLog(entry);
entries.push(rawLog);
if (entry.isGlobal) {
globalEntries.push(rawLog);
}
if (entries.length === ENTRIES_TO_BUFFER)
await insertEntries();
}
await insertEntries();
if (entriesLogged)
process.stdout.write("\n");
}
if (!Config.nofswriting)
console.log(`Writing the global modlog...`);
await this.writeFile(`${this.textLogDir}/modlog_global.txt`, globalEntries.join(""));
}
async writeFile(path, text) {
if (this.isTesting) {
const old = this.isTesting.files.get(path);
return this.isTesting.files.set(path, `${old || ""}${text}`);
}
return (0, import_lib.FS)(path).append(text);
}
}
class ModlogConverterTxt {
constructor(databaseFile, textLogDir, isTesting, newestAllowedTimestamp) {
this.isTesting = null;
this.databaseFile = databaseFile;
this.textLogDir = textLogDir;
if (isTesting || Config.nofswriting) {
this.isTesting = {
files: isTesting || /* @__PURE__ */ new Map()
};
}
this.modlog = new Modlog(
this.isTesting ? ":memory:" : this.databaseFile,
// wait 15 seconds for DB to no longer be busy - this is important since I'm trying to do
// a no-downtime transfer of text -> SQLite
{ sqliteOptions: { timeout: 15e3 } }
);
this.newestAllowedTimestamp = newestAllowedTimestamp;
}
async toSQLite() {
await this.modlog.readyPromise;
const files = this.isTesting ? [...this.isTesting.files.keys()] : await (0, import_lib.FS)(this.textLogDir).readdir();
if (files.includes("modlog_global.txt")) {
files.splice(files.indexOf("modlog_global.txt"), 1);
files.unshift("modlog_global.txt");
}
const globalEntries = {};
for (const file of files) {
if (file === "README.md")
continue;
const roomid = file.slice(7, -4);
const lines = this.isTesting ? this.isTesting.files.get(file)?.split("\n") || [] : (0, import_lib.FS)(`${this.textLogDir}/${file}`).createReadStream().byLine();
let entriesLogged = 0;
let lastLine = void 0;
let entries = [];
const insertEntries = async () => {
await this.modlog.writeSQL(entries);
entriesLogged += entries.length;
if (!Config.nofswriting) {
process.stdout.clearLine(0);
process.stdout.cursorTo(0);
process.stdout.write(`Inserted ${entriesLogged} entries from '${roomid}'`);
}
entries = [];
};
for await (const line of lines) {
const entry = parseModlog(line, lastLine, roomid === "global");
lastLine = line;
if (!entry)
continue;
if (this.newestAllowedTimestamp && entry.time > this.newestAllowedTimestamp)
break;
if (roomid !== "global" && globalEntries[entry.roomID]?.includes(line)) {
continue;
}
if (entry.isGlobal) {
if (!globalEntries[entry.roomID])
globalEntries[entry.roomID] = [];
globalEntries[entry.roomID].push(line);
}
entries.push(entry);
if (entries.length === ENTRIES_TO_BUFFER)
await insertEntries();
}
delete globalEntries[roomid];
await insertEntries();
if (entriesLogged)
process.stdout.write("\n");
}
return this.modlog.database;
}
}
class ModlogConverterTest {
constructor(inputDir, outputDir) {
this.inputDir = inputDir;
this.outputDir = outputDir;
}
async toTxt() {
const files = await (0, import_lib.FS)(this.inputDir).readdir();
if (files.includes("modlog_global.txt")) {
files.splice(files.indexOf("modlog_global.txt"), 1);
files.push("modlog_global.txt");
}
const globalEntries = [];
for (const file of files) {
if (file === "README.md")
continue;
const roomid = file.slice(7, -4);
let entriesLogged = 0;
let lastLine = void 0;
let entries = [];
const insertEntries = async () => {
if (roomid === "global")
return;
entriesLogged += entries.length;
if (!Config.nofswriting && (entriesLogged % ENTRIES_TO_BUFFER === 0 || entriesLogged < ENTRIES_TO_BUFFER)) {
process.stdout.clearLine(0);
process.stdout.cursorTo(0);
process.stdout.write(`Wrote ${entriesLogged} entries from '${roomid}'`);
}
await (0, import_lib.FS)(`${this.outputDir}/modlog_${roomid}.txt`).append(entries.join(""));
entries = [];
};
const readStream = (0, import_lib.FS)(`${this.inputDir}/${file}`).createReadStream();
for await (const line of readStream.byLine()) {
const entry = parseModlog(line, lastLine, roomid === "global");
lastLine = line;
if (!entry)
continue;
const rawLog = rawifyLog(entry);
if (roomid !== "global")
entries.push(rawLog);
if (entry.isGlobal) {
globalEntries.push(rawLog);
}
if (entries.length === ENTRIES_TO_BUFFER)
await insertEntries();
}
await insertEntries();
if (entriesLogged)
process.stdout.write("\n");
}
if (!Config.nofswriting)
console.log(`Writing the global modlog...`);
await (0, import_lib.FS)(`${this.outputDir}/modlog_global.txt`).append(globalEntries.join(""));
}
}
const ModlogConverter = {
async convert(from, to, databasePath, textLogDirectoryPath, outputLogPath, newestAllowedTimestamp) {
if (from === "txt" && to === "txt" && outputLogPath) {
const converter = new ModlogConverterTest(textLogDirectoryPath, outputLogPath);
await converter.toTxt();
console.log("\nDone!");
process.exit();
} else if (from === "sqlite" && to === "txt") {
const converter = new ModlogConverterSQLite(databasePath, textLogDirectoryPath, void 0, newestAllowedTimestamp);
await converter.toTxt();
console.log("\nDone!");
process.exit();
} else if (from === "txt" && to === "sqlite") {
const converter = new ModlogConverterTxt(databasePath, textLogDirectoryPath, void 0, newestAllowedTimestamp);
await converter.toSQLite();
console.log("\nDone!");
process.exit();
}
}
};
//# sourceMappingURL=converter.js.map
+7
View File
@@ -0,0 +1,7 @@
{
"version": 3,
"sources": ["../../../../tools/modlog/converter.ts"],
"sourcesContent": ["\r\n/**\r\n * Converts modlogs between text and SQLite; also modernizes old-format modlogs\r\n * @author Annika\r\n * @author jetou\r\n */\r\n\r\nif (!global.Config) {\r\n\tlet hasSQLite = true;\r\n\ttry {\r\n\t\trequire.resolve('better-sqlite3');\r\n\t} catch {\r\n\t\tconsole.warn(`Warning: the modlog conversion script is running without a SQLite library.`);\r\n\t\thasSQLite = false;\r\n\t}\r\n\tglobal.Config = {\r\n\t\tnofswriting: false,\r\n\t\tusesqlitemodlog: hasSQLite,\r\n\t\tusesqlite: hasSQLite,\r\n\t};\r\n}\r\n\r\nimport type * as DatabaseType from 'better-sqlite3';\r\nimport type {ModlogEntry} from '../../server/modlog';\r\nimport {FS} from '../../lib';\r\nimport {IPTools} from '../../server/ip-tools';\r\n\r\nconst Database = Config.usesqlite ? require('better-sqlite3') : null;\r\nconst {Modlog} = require('../../server/modlog');\r\n\r\ntype ModlogFormat = 'txt' | 'sqlite';\r\n\r\n/** The number of modlog entries to write to the database on each transaction */\r\nconst ENTRIES_TO_BUFFER = 7500;\r\nconst ALTS_REGEX = /\\(.*?'s (lock|mut|bann|blacklist)ed alts: (.*)\\)/;\r\nconst AUTOCONFIRMED_REGEX = /\\(.*?'s ac account: (.*)\\)/;\r\n\r\nconst IP_ONLY_ACTIONS = new Set([\r\n\t'SHAREDIP', 'UNSHAREDIP', 'UNLOCKIP', 'UNLOCKRANGE', 'RANGEBAN', 'RANGELOCK',\r\n]);\r\n\r\nexport function parseBrackets(line: string, openingBracket: '(' | '[', greedy?: boolean) {\r\n\tconst brackets = {\r\n\t\t'(': ')',\r\n\t\t'[': ']',\r\n\t};\r\n\tconst bracketOpenIndex = line.indexOf(openingBracket);\r\n\tconst bracketCloseIndex = greedy ? line.lastIndexOf(brackets[openingBracket]) : line.indexOf(brackets[openingBracket]);\r\n\tif (bracketCloseIndex < 0 || bracketOpenIndex < 0) return '';\r\n\treturn line.slice(bracketOpenIndex + 1, bracketCloseIndex);\r\n}\r\n\r\nfunction toID(text: any): ID {\r\n\treturn (text && typeof text === \"string\" ? text : \"\").toLowerCase().replace(/[^a-z0-9]+/g, \"\") as ID;\r\n}\r\n\r\nexport function modernizeLog(line: string, nextLine?: string): string | undefined {\r\n\t// first we save and remove the timestamp and the roomname\r\n\tconst prefix = line.match(/\\[.+?\\] \\(.+?\\) /i)?.[0];\r\n\tif (!prefix) return;\r\n\tif (ALTS_REGEX.test(line) || AUTOCONFIRMED_REGEX.test(line)) return;\r\n\tline = line.replace(prefix, '');\r\n\t// handle duplicate room bug\r\n\tif (line.startsWith('(')) line = line.replace(/\\([a-z0-9-]*\\) /, '');\r\n\r\n\tif (line.startsWith('(') && line.endsWith(')')) {\r\n\t\tline = line.slice(1, -1);\r\n\t}\r\n\tconst getAlts = () => {\r\n\t\tlet alts = '';\r\n\t\tnextLine?.replace(ALTS_REGEX, (_a, _b, rawAlts) => {\r\n\t\t\tif (rawAlts) alts = `alts: [${rawAlts.split(',').map(toID).join('], [')}] `;\r\n\t\t\treturn '';\r\n\t\t});\r\n\t\treturn alts;\r\n\t};\r\n\r\n\tconst getAutoconfirmed = () => {\r\n\t\tlet autoconfirmed = '';\r\n\t\tnextLine?.replace(AUTOCONFIRMED_REGEX, (_a, rawAutoconfirmed) => {\r\n\t\t\tif (rawAutoconfirmed) autoconfirmed = `ac: [${toID(rawAutoconfirmed)}] `;\r\n\t\t\treturn '';\r\n\t\t});\r\n\t\treturn autoconfirmed;\r\n\t};\r\n\r\n\t// Special cases\r\n\r\n\tif (line.startsWith('SCAV ')) {\r\n\t\tline = line.replace(/: (\\[room: .*?\\]) by (.*)/, (match, roominfo, rest) => `: by ${rest} ${roominfo}`);\r\n\t}\r\n\tline = line.replace(\r\n\t\t/(GIVEAWAY WIN|GTS FINISHED): ([A-Za-z0-9].*?)(won|has finished)/,\r\n\t\t(match, action, user) => `${action}: [${toID(user)}]:`\r\n\t);\r\n\r\n\tif (line.includes(':')) {\r\n\t\tconst possibleModernAction = line.slice(0, line.indexOf(':')).trim();\r\n\t\tif (possibleModernAction === possibleModernAction.toUpperCase()) {\r\n\t\t\tif (possibleModernAction.includes('[')) {\r\n\t\t\t\t// for corrupted lines\r\n\t\t\t\tconst [drop, ...keep] = line.split('[');\r\n\t\t\t\tprocess.stderr.write(`Ignoring malformed line: ${drop}\\n`);\r\n\t\t\t\treturn modernizeLog(keep.join(''));\r\n\t\t\t}\r\n\t\t\tif (/\\(.+\\) by [a-z0-9]{1,19}$/.test(line) && !['OLD MODLOG', 'NOTE'].includes(possibleModernAction)) {\r\n\t\t\t\t// weird reason formatting\r\n\t\t\t\tconst reason = parseBrackets(line, '(', true);\r\n\t\t\t\treturn `${prefix}${line.replace(` (${reason})`, '')}: ${reason}`;\r\n\t\t\t}\r\n\t\t\t// Log is already modernized\r\n\t\t\treturn `${prefix}${line}`;\r\n\t\t}\r\n\t}\r\n\r\n\tif (/\\[(the|a)poll\\] was (started|ended) by/.test(line)) {\r\n\t\tconst actionTaker = toID(line.slice(line.indexOf(' by ') + ' by '.length));\r\n\t\tconst isEnding = line.includes('was ended by');\r\n\t\treturn `${prefix}POLL${isEnding ? ' END' : ''}: by ${actionTaker}`;\r\n\t}\r\n\tif (/User (.*?) won the game of (.*?) mode trivia/.test(line)) {\r\n\t\treturn `${prefix}TRIVIAGAME: by unknown: ${line}`;\r\n\t}\r\n\r\n\tconst modernizerTransformations: {[k: string]: (log: string) => string} = {\r\n\t\t'notes: ': (log) => {\r\n\t\t\tconst [actionTaker, ...rest] = line.split(' notes: ');\r\n\t\t\treturn `NOTE: by ${toID(actionTaker)}: ${rest.join('')}`;\r\n\t\t},\r\n\r\n\t\t' declared': (log) => {\r\n\t\t\tletLine truncated
"mappings": ";;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAwBA,iBAAiB;AACjB,sBAAsB;AAlBtB,IAAI,CAAC,OAAO,QAAQ;AACnB,MAAI,YAAY;AAChB,MAAI;AACH,oBAAgB,gBAAgB;AAAA,EACjC,QAAE;AACD,YAAQ,KAAK,4EAA4E;AACzF,gBAAY;AAAA,EACb;AACA,SAAO,SAAS;AAAA,IACf,aAAa;AAAA,IACb,iBAAiB;AAAA,IACjB,WAAW;AAAA,EACZ;AACD;AAOA,MAAM,WAAW,OAAO,YAAY,QAAQ,gBAAgB,IAAI;AAChE,MAAM,EAAC,OAAM,IAAI,QAAQ,qBAAqB;AAK9C,MAAM,oBAAoB;AAC1B,MAAM,aAAa;AACnB,MAAM,sBAAsB;AAE5B,MAAM,kBAAkB,oBAAI,IAAI;AAAA,EAC/B;AAAA,EAAY;AAAA,EAAc;AAAA,EAAY;AAAA,EAAe;AAAA,EAAY;AAClE,CAAC;AAEM,SAAS,cAAc,MAAc,gBAA2B,QAAkB;AACxF,QAAM,WAAW;AAAA,IAChB,KAAK;AAAA,IACL,KAAK;AAAA,EACN;AACA,QAAM,mBAAmB,KAAK,QAAQ,cAAc;AACpD,QAAM,oBAAoB,SAAS,KAAK,YAAY,SAAS,cAAc,CAAC,IAAI,KAAK,QAAQ,SAAS,cAAc,CAAC;AACrH,MAAI,oBAAoB,KAAK,mBAAmB;AAAG,WAAO;AAC1D,SAAO,KAAK,MAAM,mBAAmB,GAAG,iBAAiB;AAC1D;AAEA,SAAS,KAAK,MAAe;AAC5B,UAAQ,QAAQ,OAAO,SAAS,WAAW,OAAO,IAAI,YAAY,EAAE,QAAQ,eAAe,EAAE;AAC9F;AAEO,SAAS,aAAa,MAAc,UAAuC;AAEjF,QAAM,SAAS,KAAK,MAAM,mBAAmB,IAAI,CAAC;AAClD,MAAI,CAAC;AAAQ;AACb,MAAI,WAAW,KAAK,IAAI,KAAK,oBAAoB,KAAK,IAAI;AAAG;AAC7D,SAAO,KAAK,QAAQ,QAAQ,EAAE;AAE9B,MAAI,KAAK,WAAW,GAAG;AAAG,WAAO,KAAK,QAAQ,mBAAmB,EAAE;AAEnE,MAAI,KAAK,WAAW,GAAG,KAAK,KAAK,SAAS,GAAG,GAAG;AAC/C,WAAO,KAAK,MAAM,GAAG,EAAE;AAAA,EACxB;AACA,QAAM,UAAU,MAAM;AACrB,QAAI,OAAO;AACX,cAAU,QAAQ,YAAY,CAAC,IAAI,IAAI,YAAY;AAClD,UAAI;AAAS,eAAO,UAAU,QAAQ,MAAM,GAAG,EAAE,IAAI,IAAI,EAAE,KAAK,MAAM;AACtE,aAAO;AAAA,IACR,CAAC;AACD,WAAO;AAAA,EACR;AAEA,QAAM,mBAAmB,MAAM;AAC9B,QAAI,gBAAgB;AACpB,cAAU,QAAQ,qBAAqB,CAAC,IAAI,qBAAqB;AAChE,UAAI;AAAkB,wBAAgB,QAAQ,KAAK,gBAAgB;AACnE,aAAO;AAAA,IACR,CAAC;AACD,WAAO;AAAA,EACR;AAIA,MAAI,KAAK,WAAW,OAAO,GAAG;AAC7B,WAAO,KAAK,QAAQ,6BAA6B,CAAC,OAAO,UAAU,SAAS,QAAQ,QAAQ,UAAU;AAAA,EACvG;AACA,SAAO,KAAK;AAAA,IACX;AAAA,IACA,CAAC,OAAO,QAAQ,SAAS,GAAG,YAAY,KAAK,IAAI;AAAA,EAClD;AAEA,MAAI,KAAK,SAAS,GAAG,GAAG;AACvB,UAAM,uBAAuB,KAAK,MAAM,GAAG,KAAK,QAAQ,GAAG,CAAC,EAAE,KAAK;AACnE,QAAI,yBAAyB,qBAAqB,YAAY,GAAG;AAChE,UAAI,qBAAqB,SAAS,GAAG,GAAG;AAEvC,cAAM,CAAC,MAAM,GAAG,IAAI,IAAI,KAAK,MAAM,GAAG;AACtC,gBAAQ,OAAO,MAAM,4BAA4B;AAAA,CAAQ;AACzD,eAAO,aAAa,KAAK,KAAK,EAAE,CAAC;AAAA,MAClC;AACA,UAAI,4BAA4B,KAAK,IAAI,KAAK,CAAC,CAAC,cAAc,MAAM,EAAE,SAAS,oBAAoB,GAAG;AAErG,cAAM,SAAS,cAAc,MAAM,KAAK,IAAI;AAC5C,eAAO,GAAG,SAAS,KAAK,QAAQ,KAAK,WAAW,EAAE,MAAM;AAAA,MACzD;AAEA,aAAO,GAAG,SAAS;AAAA,IACpB;AAAA,EACD;AAEA,MAAI,yCAAyC,KAAK,IAAI,GAAG;AACxD,UAAM,cAAc,KAAK,KAAK,MAAM,KAAK,QAAQ,MAAM,IAAI,OAAO,MAAM,CAAC;AACzE,UAAM,WAAW,KAAK,SAAS,cAAc;AAC7C,WAAO,GAAG,aAAa,WAAW,SAAS,UAAU;AAAA,EACtD;AACA,MAAI,+CAA+C,KAAK,IAAI,GAAG;AAC9D,WAAO,GAAG,iCAAiC;AAAA,EAC5C;AAEA,QAAM,4BAAoE;AAAA,IACzE,WAAW,CAAC,QAAQ;AACnB,YAAM,CAAC,aAAa,GAAG,IAAI,IAAI,KAAK,MAAM,UAAU;AACpD,aAAO,YAAY,KAAK,WAAW,MAAM,KAAK,KAAK,EAAE;AAAA,IACtD;AAAA,IAEA,aAAa,CAAC,QAAQ;AACrB,UAAI,YAAY;AAChB,UAAI,YAAY;AAChB,UAAI,IAAI,SAAS,oBAAoB,GAAG;AACvC,oBAAY;AACZ,oBAAY;AAAA,MACb;AACA,UAAI,IAAI,SAAS,cAAc,GAAG;AACjC,qBAAa;AACb,oBAAY;AAAA,MACb;AAEA,YAAM,kBAAkB,KAAK,IAAI,MAAM,GAAG,IAAI,YAAY,SAAS,CAAC,CAAC;AAErE,YAAM,IAAI,MAAM,gBAAgB,MAAM;AACtC,YAAM,IAAI,MAAM,UAAU,MAAM;AAChC,YAAM,IAAI,QAAQ,SAAS,EAAE,EAAE,KAAK;AACpC,aAAO,GAAG,iBAAiB,oBAAoB;AAAA,IAChD;AAAA,IAEA,6BAA6B,CAAC,QAAQ;AACrC,YAAM,cAAc,cAAc,KAAK,GAAG;AAC1C,YAAM,IAAI,MAAM,YAAY,SAAS,CAAC;AACtC,YAAM,IAAI,MAAM,4BAA4B,SAAS,GAAG,EAAE;AAC1D,aAAO,gBAAgB,oBAAoB;AAAA,IAC5C;AAAA,IAEA,sBAAsB,CAAC,QAAQ;AAC9B,UAAI;AACJ,UAAI,IAAI,SAAS,6BAA6B,GAAG;AAChD,iBAAS;AAAA,MACV,WAAW,IAAI,SAAS,+BAA+B,GAAG;AACzD,iBAAS;AAAA,MACV,OAAO;AACN,iBAAS;AAAA,MACV;AACA,YAAM,kBAAkB,IAAI,MAAM,GAAG,IAAI,YAAY,IAAI,2BAA2B,CAAC;AACrF,YAAM,IAAI,MAAM,gBAAgB,SAAS,CAAC;AAC1C,YAAM,YAAY,IAAI,MAAM,IAAI,2BAA2B,QAAQ,EAAE;AACrE,aAAO,iBAAiB,KAAK,eAAe,MAAM,OAAO,MAAM,GAAG,EAAE,CAAC,MAAM;AAAA,IAC5E;AAAA,IAEA,mBAAmB,CAAC,QAAQ;AAC3B,YAAM,cAAc,cAAc,KAAK,GAAG;AAC1C,YAAM,IAAI,MAAM,YAAY,SAAS,CAAC;AACtC,YAAM,IAAI,MAAM,kBAAkB,MAAM;AACxC,aAAO,eAAe,mBAAmB;AAAA,IAC1C;AAAA,IACA,mBAAmB,CAAC,QAAQ;AAC3B,YAAM,kBAAkB,IAAI,MAAM,GAAG,IAAI,YAAY,MAAM,CAAC;AAC5D,YAAM,IAAI,MAAM,gBAAgB,SAAS,CAAC;AAC1C,YAAM,IAAI,MAAM,kBAAkB,MAAM;AACxC,YAAM,OAAO,IAAI,WAAW,MAAM,IAAI,SAAS,IAAI,QAAQ,KAAK,EAAE;AAClE,aAAO,UAAU,SAAS,SAAS,UAAU,UAAU,KAAK,eAAe,IAAI,SAAS,SAAS,KAAK,SAAS;AAAA,IAChH;AAAA,IACA,sBAAsB,CAAC,QAAQ;AAC9B,YAAM,kBAAkB,IAAI,MAAM,GAAG,IAAI,YAAY,qBAAqB,CAAC;AAC3E,aAAO,eAAe,KAAK,eAAe;AAAA,IAC3C;AAAA,IAEA,yBAAyB,CAAC,QAAQ;AACjC,YAAM,aAAa,gCAAgC,KAAK,GAAG;AAC3D,YAAM,cAAc,IAAI,SAAS,WAAW;AAC5C,YAAM,cAAc,KAAK,IAAI,MAAM,GAAG,IAAI,QAAQ,aAAa,YAAY,SAAS,CAAC,CAAC;AACtF,aAAO,GAAG,aAAa,WAAW,KAAK,cAAc,SAAS,oBAAoB;AAAA,IACnF;AAAA,IACA,yBAAyB,CAAC,QAAQ,0BAA0B,uBAAuB,EAAE,GAAG;AAAA,IACxF,0BAA0B,CAAC,QAAQ,0BAA0B,uBAAuB,EAAE,GAAG;AAAA,IACzF,0BAA0B,CAAC,QAAQ,0BAA0B,uBAAuB,EAAE,GAAG;AAAA,IAEzF,2BAA2B,CAAC,QAAQ;AACnC,YAAM,cAAc,cAAc,KAAK,GAAG;AAC1C,YAAM,IAAI,MAAM,YAAY,SAAS,CAAC;AACtC,YAAM,IAAI,MAAM,IAAI,EAAE;AACtB,aAAO,mBAAmB,gBAAgB;AAAA,IAC3C;AAAA,IACA,2CAA2C,CAAC,QAAQ;AACnD,YAAM,eAAe,cAAc,KAAK,GAAG;AAC3C,YAAM,IAAI,MAAM,aAAa,SAAS,CAAC;AACvC,YAAM,IAAI,MAAM,0CAA0C,MAAM;AAChE,aAAO,aAAa,KAAK,YAAY,SAAS,KAAK,GAAG;AAAA,IACvD;AAAA,IACA,qDAAqD,CAAC,QAAQLine truncated
"names": []
}
+590
View File
@@ -0,0 +1,590 @@
"use strict";
var __create = Object.create;
var __defProp = Object.defineProperty;
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
var __getOwnPropNames = Object.getOwnPropertyNames;
var __getProtoOf = Object.getPrototypeOf;
var __hasOwnProp = Object.prototype.hasOwnProperty;
var __export = (target, all) => {
for (var name in all)
__defProp(target, name, { get: all[name], enumerable: true });
};
var __copyProps = (to, from, except, desc) => {
if (from && typeof from === "object" || typeof from === "function") {
for (let key of __getOwnPropNames(from))
if (!__hasOwnProp.call(to, key) && key !== except)
__defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
}
return to;
};
var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
// If the importer is in node compatibility mode or this is not an ESM
// file that has been converted to a CommonJS file using a Babel-
// compatible transform (i.e. "__esModule" has not been set), then set
// "default" to the CommonJS "module.exports" for node compatibility.
isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
mod
));
var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
var importer_exports = {};
__export(importer_exports, {
TIERS: () => TIERS,
fetch: () => fetch,
getStatisticsURL: () => getStatisticsURL,
importAll: () => importAll
});
module.exports = __toCommonJS(importer_exports);
var http = __toESM(require("http"));
var https = __toESM(require("https"));
var url = __toESM(require("url"));
var util = __toESM(require("util"));
var smogon = __toESM(require("smogon"));
var import_lib = require("../../lib");
var import_dex = require("../../sim/dex");
var import_team_validator = require("../../sim/team-validator");
import_dex.Dex.includeModData();
const TIERS = /* @__PURE__ */ new Set([
"ubers",
"ou",
"uu",
"ru",
"nu",
"pu",
"zu",
"lc",
"cap",
"nationaldex",
"doublesou",
"battlespotsingles",
"battlespotdoubles",
"battlestadiumsingles",
"vgc2016",
"vgc2017",
"vgc2018",
"vgc2019ultraseries",
"vgc2020",
"1v1",
"anythinggoes",
"nationaldexag",
"balancedhackmons",
"letsgoou",
"monotype",
"purehackmons"
]);
const FORMATS = /* @__PURE__ */ new Map();
const VALIDATORS = /* @__PURE__ */ new Map();
for (let gen = 1; gen <= 9; gen++) {
for (const tier of TIERS) {
const format = import_dex.Dex.formats.get(`gen${gen}${tier}`);
if (format.exists) {
FORMATS.set(format.id, { gen, format });
VALIDATORS.set(format.id, new import_team_validator.TeamValidator(format));
}
}
}
async function importAll() {
const index = await request(smogon.Statistics.URL);
const imports = [];
for (let gen = 1; gen <= 9; gen++) {
imports.push(importGen(gen, index));
}
return Promise.all(imports);
}
async function importGen(gen, index) {
const data = {};
const smogonSetsByFormat = {};
const thirdPartySetsByFormat = {};
const numByFormat = {};
const imports = [];
const dex = import_dex.Dex.forFormat(`gen${gen}ou`);
for (const id in dex.data.Pokedex) {
if (!eligible(dex, id))
continue;
const species = dex.species.get(id);
if (species.battleOnly)
continue;
imports.push(importSmogonSets(dex.species.get(id).name, gen, smogonSetsByFormat, numByFormat));
}
await Promise.all(imports);
for (const { format, gen: g } of FORMATS.values()) {
if (g !== gen)
continue;
if (smogonSetsByFormat[format.id] && Object.keys(smogonSetsByFormat[format.id]).length) {
data[format.id] = {};
data[format.id]["dex"] = smogonSetsByFormat[format.id];
report(format, numByFormat[format.id], "dex");
}
for (const source in thirdPartySetsByFormat) {
if (thirdPartySetsByFormat[source][format.id] && Object.keys(thirdPartySetsByFormat[source][format.id]).length) {
data[format.id] = data[format.id] || {};
data[format.id][source] = thirdPartySetsByFormat[source][format.id];
}
}
const stats = await getStatisticsURL(index, format);
if (!stats)
continue;
try {
const statistics = smogon.Statistics.process(await request(stats.url));
const sets = importUsageBasedSets(gen, format, statistics, stats.count);
if (Object.keys(sets).length) {
data[format.id] = data[format.id] || {};
data[format.id]["stats"] = sets;
}
data[format.id] = data[format.id] || {};
} catch (err) {
error(`${stats.url} = ${err}`);
}
}
return data;
}
function eligible(dex, id) {
const gen = toGen(dex, id);
if (!gen || gen > dex.gen)
return false;
const species = dex.species.get(id);
if (["Mega", "Primal", "Ultra"].some((f) => species.forme.startsWith(f)))
return true;
const unique = ["darmanitan", "meloetta", "greninja", "zygarde"];
const similar = ["pichu", "pikachu", "genesect", "basculin", "magearna", "keldeo", "vivillon"];
if (species.battleOnly && !unique.some((f) => id.startsWith(f)))
return false;
const capNFE = species.isNonstandard === "CAP" && species.nfe;
return !id.endsWith("totem") && !capNFE && !similar.some((f) => id.startsWith(f) && id !== f);
}
function toGen(dex, name) {
const pokemon = dex.species.get(name);
if (pokemon.isNonstandard === "LGPE")
return 7;
if (!pokemon.exists || pokemon.isNonstandard && pokemon.isNonstandard !== "CAP")
return void 0;
const n = pokemon.num;
if (n > 905)
return 9;
if (n > 810)
return 8;
if (n > 721 || n <= -23 && n >= -28 || n <= -120 && n >= -126)
return 7;
if (n > 649 || n <= -8 && n >= -22 || n <= -106 && n >= -110)
return 6;
if (n > 493 || n <= -12 && n >= -17 || n <= -111 && n >= -115)
return 5;
if (n > 386 || n <= -1 && n >= -11 || n <= -101 && n >= -104 || n <= -116 && n >= -119)
return 4;
if (n > 251)
return 3;
if (n > 151)
return 2;
if (n > 0)
return 1;
}
async function importSmogonSets(pokemon, gen, setsByFormat, numByFormat) {
const analysesByFormat = await getAnalysesByFormat(pokemon, gen);
if (!analysesByFormat)
return;
for (const [format, analyses] of analysesByFormat.entries()) {
const dex = import_dex.Dex.forFormat(format);
let setsForPokemon = setsByFormat[format.id];
if (!setsForPokemon) {
setsForPokemon = {};
setsByFormat[format.id] = setsForPokemon;
}
let baseSpecies = dex.species.get(pokemon);
if (baseSpecies.baseSpecies !== baseSpecies.name)
baseSpecies = dex.species.get(baseSpecies.baseSpecies);
const battleOnlyFormes = [];
if (baseSpecies.otherFormes) {
for (const forme of baseSpecies.otherFormes) {
const formeSpecies = dex.species.get(forme);
if (formeSpecies.battleOnly && eligible(dex, (0, import_dex.toID)(formeSpecies))) {
battleOnlyFormes.push(formeSpecies);
}
}
}
for (const analysis of analyses) {
for (const moveset of analysis.movesets) {
const set = movesetToPokemonSet(dex, format, pokemon, moveset);
const name = cleanName(moveset.name);
addSmogonSet(dex, format, pokemon, name, set, setsForPokemon, numByFormat);
for (const battleOnlyForme of battleOnlyFormes) {
const s = { ...set };
if (!format.id.includes("balancedhackmons"))
s.ability = battleOnlyForme.abilities[0];
if (typeof battleOnlyForme.battleOnly !== "string") {
if (!battleOnlyForme.battleOnly.includes(pokemon))
continue;
const species = dex.species.get(pokemon);
const disambiguated = `${name} - ${species.baseForme || species.forme}`;
addSmogonSet(dex, format, battleOnlyForme.name, disambiguated, s, setsForPokemon, numByFormat, pokemon);
} else if (battleOnlyForme.battleOnly === pokemon) {
addSmogonSet(dex, format, battleOnlyForme.name, name, s, setsForPokemon, numByFormat);
}
}
}
}
}
}
function addSmogonSet(dex, format, pokemon, name, set, setsForPokemon, numByFormat, outOfBattleSpeciesName) {
if (validSet("dex", dex, format, pokemon, name, set, outOfBattleSpeciesName)) {
setsForPokemon[pokemon] = setsForPokemon[pokemon] || {};
setsForPokemon[pokemon][name] = set;
numByFormat[format.id] = (numByFormat[format.id] || 0) + 1;
}
}
function cleanName(name) {
return name.replace(/"/g, `'`);
}
function movesetToPokemonSet(dex, format, pokemon, set) {
const level = getLevel(format, set.levels[0]);
return {
level: level === 100 ? void 0 : level,
moves: set.moveslots.map((ms) => ms[0]).map((s) => s.type ? `${s.move} ${s.type}` : s.move),
ability: fixedAbility(dex, pokemon, set.abilities[0]),
item: set.items[0] === "No Item" ? void 0 : set.items[0],
nature: set.natures[0],
teraType: set.teratypes ? set.teratypes[0] : void 0,
ivs: toStatsTable(set.ivconfigs[0], 31),
evs: toStatsTable(set.evconfigs[0])
};
}
function toStatsTable(stats, elide = 0) {
if (!stats)
return void 0;
const s = {};
let stat;
for (stat in stats) {
const val = stats[stat];
if (val !== elide)
s[stat] = val;
}
return s;
}
function fixedAbility(dex, pokemon, ability) {
if (dex.gen <= 2)
return void 0;
const species = dex.species.get(pokemon);
if (ability && !["Mega", "Primal", "Ultra"].some((f) => species.forme.startsWith(f)))
return ability;
return species.abilities[0];
}
function validSet(source, dex, format, pokemon, name, set, outOfBattleSpeciesName) {
if (skip(dex, format, pokemon, set))
return false;
const pset = toPokemonSet(dex, format, pokemon, set, outOfBattleSpeciesName);
let invalid = VALIDATORS.get(format.id).validateSet(pset, {});
if (!invalid)
return true;
if (invalid.length === 1 && invalid[0].includes("must be shiny")) {
set.shiny = true;
pset.shiny = true;
invalid = VALIDATORS.get(format.id).validateSet(pset, {});
if (!invalid)
return true;
}
if (format.id === "gen4ubers" && invalid.includes(`${pokemon} is banned.`))
return true;
const title = `${format.name}: ${pokemon} (${name})'`;
const details = `${JSON.stringify(set)} = ${invalid.join(", ")}`;
console.error(color(`${source} Invalid set ${title}: ${details}`, 90));
return false;
}
function skip(dex, format, pokemon, set) {
const { gen } = FORMATS.get(format.id);
const hasMove = (m) => set.moves && set.moves.includes(m);
const bh = format.id.includes("balancedhackmons");
if (pokemon === "Groudon-Primal" && set.item !== "Red Orb")
return true;
if (pokemon === "Kyogre-Primal" && set.item !== "Blue Orb" && !(bh && gen === 7))
return true;
if (bh)
return false;
if (dex.species.get(pokemon).forme.startsWith("Mega")) {
if (pokemon === "Rayquaza-Mega") {
return format.id.includes("ubers") || !hasMove("Dragon Ascent");
} else {
return dex.items.get(set.item).megaStone !== pokemon;
}
}
if (pokemon === "Necrozma-Ultra" && set.item !== "Ultranecrozium Z")
return true;
if (pokemon === "Greninja-Ash" && set.ability !== "Battle Bond")
return true;
if (pokemon === "Zygarde-Complete" && set.ability !== "Power Construct")
return true;
if (pokemon === "Darmanitan-Zen" && set.ability !== "Zen Mode")
return true;
if (pokemon === "Meloetta-Pirouette" && !hasMove("Relic Song"))
return true;
return false;
}
function toPokemonSet(dex, format, pokemon, set, outOfBattleSpeciesName) {
const hp = set.moves && set.moves.find((m) => m.startsWith("Hidden Power"));
let fill = dex.gen === 2 ? 30 : 31;
if (hp) {
const type = hp.slice(13);
if (type && dex.getHiddenPower(fillStats(set.ivs, fill)).type !== type) {
if (!set.ivs || dex.gen >= 7 && (!set.level || set.level === 100)) {
set.hpType = type;
fill = 31;
} else if (dex.gen === 2) {
const dvs = { ...dex.types.get(type).HPdvs };
let stat;
for (stat in dvs) {
dvs[stat] *= 2;
}
set.ivs = { ...dvs, ...set.ivs };
set.ivs.hp = expectedHP(set.ivs);
} else {
set.ivs = { ...dex.types.get(type).HPivs, ...set.ivs };
}
}
}
const copy = { species: pokemon, ...set };
copy.ivs = fillStats(set.ivs, fill);
if (!set.evs && dex.gen >= 3 && format.id !== "gen7letsgoou")
set.evs = { spe: 1 };
copy.evs = fillStats(set.evs, dex.gen <= 2 ? 252 : 0);
copy.ability = copy.ability || "None";
const species = dex.species.get(pokemon);
if (species.battleOnly && !format.id.includes("balancedhackmons")) {
if (outOfBattleSpeciesName) {
copy.species = outOfBattleSpeciesName;
} else if (typeof species.battleOnly === "string") {
copy.species = species.battleOnly;
} else {
throw new Error(`Unable to disambiguate out of battle species for ${species.name} in ${format.id}`);
}
copy.ability = dex.species.get(copy.species).abilities[0];
}
return copy;
}
function expectedHP(ivs) {
ivs = fillStats(ivs, 31);
const atkDV = Math.floor(ivs.atk / 2);
const defDV = Math.floor(ivs.def / 2);
const speDV = Math.floor(ivs.spe / 2);
const spcDV = Math.floor(ivs.spa / 2);
return 2 * (atkDV % 2 * 8 + defDV % 2 * 4 + speDV % 2 * 2 + spcDV % 2);
}
function fillStats(stats, fill = 0) {
return import_team_validator.TeamValidator.fillStats(stats || null, fill);
}
const SMOGON = {
uber: "ubers",
doubles: "doublesou",
lgpeou: "letsgoou",
ag: "anythinggoes",
bh: "balancedhackmons",
vgc16: "vgc2016",
vgc17: "vgc2017",
vgc18: "vgc2018",
vgc19: "vgc2019ultraseries"
};
const getAnalysis = retrying(async (u) => {
try {
return smogon.Analyses.process(await request(u));
} catch (err) {
if (err.message.startsWith("HTTP")) {
return Promise.reject(err);
} else {
return Promise.reject(new RetryableError(err.message));
}
}
}, 3, 50);
async function getAnalysesByFormat(pokemon, gen) {
const u = smogon.Analyses.url(pokemon === "Meowstic" ? "Meowstic-M" : pokemon, gen);
try {
const analysesByTier = await getAnalysis(u);
if (!analysesByTier) {
error(`Unable to process analysis for ${pokemon} in generation ${gen}`);
return void 0;
}
const analysesByFormat = /* @__PURE__ */ new Map();
for (const [tier, analyses] of analysesByTier.entries()) {
const t = (0, import_dex.toID)(tier);
const f = FORMATS.get(`gen${gen}${SMOGON[t] || t}`);
if (f)
analysesByFormat.set(f.format, analyses);
}
return analysesByFormat;
} catch {
error(`Unable to process analysis for ${pokemon} in generation ${gen}`);
return void 0;
}
}
function getLevel(format, level = 0) {
const ruleTable = import_dex.Dex.formats.getRuleTable(format);
if (ruleTable.adjustLevel)
return ruleTable.adjustLevel;
const maxLevel = ruleTable.maxLevel;
const adjustLevelDown = ruleTable.adjustLevelDown || maxLevel;
if (!level)
level = ruleTable.defaultLevel;
return level > adjustLevelDown ? adjustLevelDown : level;
}
async function getStatisticsURL(index, format) {
const current = index.includes(format.id);
const latest = await smogon.Statistics.latestDate(format.id, !current);
if (!latest)
return void 0;
return { url: smogon.Statistics.url(latest.date, format.id, current || 1500), count: latest.count };
}
function importUsageBasedSets(gen, format, statistics, count) {
const sets = {};
const dex = import_dex.Dex.forFormat(format);
const threshold = getUsageThreshold(format, count);
let num = 0;
for (const pokemon in statistics.data) {
const stats = statistics.data[pokemon];
if (eligible(dex, (0, import_dex.toID)(pokemon)) && stats.usage >= threshold) {
const set = {
level: getLevel(format),
moves: top(stats.Moves, 4).map((m) => dex.moves.get(m).name).filter((m) => m)
};
if (gen >= 2 && format.id !== "gen7letsgoou") {
const id = top(stats.Items);
set.item = dex.items.get(id).name;
if (set.item === "nothing")
set.item = void 0;
}
if (gen >= 3) {
const id = top(stats.Abilities);
set.ability = fixedAbility(dex, pokemon, dex.abilities.get(id).name);
const { nature, evs } = fromSpread(top(stats.Spreads));
set.nature = nature;
if (format.id !== "gen7letsgoou") {
if (!evs || !Object.keys(evs).length)
continue;
set.evs = evs;
}
}
const name = "Showdown Usage";
if (validSet("stats", dex, format, pokemon, name, set)) {
sets[pokemon] = {};
sets[pokemon][name] = set;
num++;
}
}
}
report(format, num, "stats");
return sets;
}
function getUsageThreshold(format, count) {
if (count < 100)
return Infinity;
if (count < 400)
return 0.05;
return /uber|anythinggoes|doublesou/.test(format.id) ? 0.03 : 0.01;
}
const STATS = import_dex.Dex.stats.ids();
function fromSpread(spread) {
const [nature, revs] = spread.split(":");
const evs = {};
for (const [i, rev] of revs.split("/").entries()) {
const ev = Number(rev);
if (ev)
evs[STATS[i]] = ev;
}
return { nature, evs };
}
function top(weighted, n = 1) {
if (n === 0)
return void 0;
if (n === 1) {
let max;
for (const key in weighted) {
if (!max || weighted[max] < weighted[key])
max = key;
}
return max;
}
return Object.entries(weighted).sort((a, b) => b[1] - a[1]).slice(0, n).map((x) => x[0]);
}
class RetryableError extends Error {
constructor(message) {
super(message);
Object.setPrototypeOf(this, new.target.prototype);
}
}
const request = retrying(throttling(fetch, 1, 50), 5, 20);
function fetch(u) {
const client = u.startsWith("http:") ? http : https;
return new Promise((resolve, reject) => {
const req = client.get(u, (res) => {
if (res.statusCode !== 200) {
if (res.statusCode >= 500 && res.statusCode < 600) {
return reject(new RetryableError(`HTTP ${res.statusCode}`));
} else if (res.statusCode >= 300 && res.statusCode <= 400 && res.headers.location) {
resolve(fetch(url.resolve(u, res.headers.location)));
} else {
return reject(new Error(`HTTP ${res.statusCode}`));
}
}
import_lib.Streams.readAll(res).then(resolve, reject);
});
req.on("error", reject);
req.end();
});
}
function retrying(fn, retries, wait) {
const retry = async (args, attempt = 0) => {
try {
return await fn(args);
} catch (err) {
if (err instanceof RetryableError) {
attempt++;
if (attempt > retries)
return Promise.reject(err);
const timeout = Math.round(attempt * wait * (1 + Math.random() / 2));
warn(`Retrying ${args} in ${timeout}ms (${attempt}):`, err);
return new Promise((resolve) => {
setTimeout(() => {
resolve(retry(args, attempt++));
}, timeout);
});
} else {
return Promise.reject(err);
}
}
};
return retry;
}
function throttling(fn, limit, interval) {
const queue = /* @__PURE__ */ new Map();
let currentTick = 0;
let activeCount = 0;
const throttled = (args) => {
let timeout;
return new Promise((resolve, reject) => {
const execute = () => {
resolve(fn(args));
queue.delete(timeout);
};
const now = Date.now();
if (now - currentTick > interval) {
activeCount = 1;
currentTick = now;
} else if (activeCount < limit) {
activeCount++;
} else {
currentTick += interval;
activeCount = 1;
}
timeout = setTimeout(execute, currentTick - now);
queue.set(timeout, reject);
});
};
return throttled;
}
function color(s, code) {
return util.format(`\x1B[${code}m%s\x1B[0m`, s);
}
function report(format, num, source) {
console.info(`${format.name}: ${color(num, 33)} ${color(`(${source})`, 90)}`);
}
function warn(s, err) {
console.warn(`${color(s, 33)} ${color(err.message, 90)}`);
}
function error(s) {
console.error(color(s, 91));
}
//# sourceMappingURL=importer.js.map
@@ -0,0 +1,7 @@
{
"version": 3,
"sources": ["../../../../tools/set-import/importer.ts"],
"sourcesContent": ["import * as http from 'http';\r\nimport * as https from 'https';\r\nimport * as url from 'url';\r\nimport * as util from 'util';\r\n\r\nimport * as smogon from 'smogon';\r\n\r\nimport {Streams} from '../../lib';\r\nimport {Dex, toID} from '../../sim/dex';\r\nimport {TeamValidator} from '../../sim/team-validator';\r\nDex.includeModData();\r\n\r\ntype DeepPartial<T> = {\r\n\t[P in keyof T]?: T[P] extends (infer I)[]\r\n\t\t? (DeepPartial<I>)[]\r\n\t\t: DeepPartial<T[P]>;\r\n};\r\n\r\ninterface PokemonSets {\r\n\t[speciesid: string]: {\r\n\t\t[name: string]: DeepPartial<PokemonSet>,\r\n\t};\r\n}\r\n\r\ninterface IncomingMessage extends NodeJS.ReadableStream {\r\n\tstatusCode: number;\r\n\theaders: {location?: string};\r\n}\r\n\r\n// eg. 'gen1.json'\r\ninterface GenerationData {\r\n\t[formatid: string]: FormatData;\r\n}\r\n\r\n// eg. 'gen7balancedhackmons.json'\r\ninterface FormatData {\r\n\t[source: string]: PokemonSets;\r\n}\r\n\r\ntype GenerationNum = 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9;\r\n\r\n// The tiers we support, ie. ones that we have data sources for.\r\nexport const TIERS = new Set([\r\n\t'ubers', 'ou', 'uu', 'ru', 'nu', 'pu', 'zu', 'lc', 'cap', 'nationaldex',\r\n\t'doublesou', 'battlespotsingles', 'battlespotdoubles', 'battlestadiumsingles',\r\n\t'vgc2016', 'vgc2017', 'vgc2018', 'vgc2019ultraseries', 'vgc2020', '1v1',\r\n\t'anythinggoes', 'nationaldexag', 'balancedhackmons', 'letsgoou', 'monotype',\r\n\t'purehackmons',\r\n]);\r\nconst FORMATS = new Map<ID, {gen: GenerationNum, format: Format}>();\r\nconst VALIDATORS = new Map<ID, TeamValidator>();\r\nfor (let gen = 1; gen <= 9; gen++) {\r\n\tfor (const tier of TIERS) {\r\n\t\tconst format = Dex.formats.get(`gen${gen}${tier}`);\r\n\t\tif (format.exists) {\r\n\t\t\tFORMATS.set(format.id, {gen: gen as GenerationNum, format});\r\n\t\t\tVALIDATORS.set(format.id, new TeamValidator(format));\r\n\t\t}\r\n\t}\r\n}\r\n\r\nexport async function importAll() {\r\n\tconst index = await request(smogon.Statistics.URL);\r\n\r\n\tconst imports = [];\r\n\tfor (let gen = 1; gen <= 9; gen++) {\r\n\t\timports.push(importGen(gen as GenerationNum, index));\r\n\t}\r\n\r\n\treturn Promise.all(imports);\r\n}\r\n\r\nasync function importGen(gen: GenerationNum, index: string) {\r\n\tconst data: GenerationData = {};\r\n\r\n\tconst smogonSetsByFormat: {[formatid: string]: PokemonSets} = {};\r\n\tconst thirdPartySetsByFormat: {[source: string]: {[formatid: string]: PokemonSets}} = {};\r\n\r\n\tconst numByFormat: {[formatid: string]: number} = {};\r\n\tconst imports = [];\r\n\tconst dex = Dex.forFormat(`gen${gen}ou`);\r\n\tfor (const id in dex.data.Pokedex) {\r\n\t\tif (!eligible(dex, id as ID)) continue;\r\n\t\tconst species = dex.species.get(id);\r\n\t\tif (species.battleOnly) continue;// Smogon collapses these into their out of battle species\r\n\t\timports.push(importSmogonSets(dex.species.get(id).name, gen, smogonSetsByFormat, numByFormat));\r\n\t}\r\n\tawait Promise.all(imports);\r\n\r\n\tfor (const {format, gen: g} of FORMATS.values()) {\r\n\t\tif (g !== gen) continue;\r\n\r\n\t\tif (smogonSetsByFormat[format.id] && Object.keys(smogonSetsByFormat[format.id]).length) {\r\n\t\t\tdata[format.id] = {};\r\n\t\t\tdata[format.id]['dex'] = smogonSetsByFormat[format.id];\r\n\t\t\treport(format, numByFormat[format.id], 'dex');\r\n\t\t}\r\n\r\n\t\tfor (const source in thirdPartySetsByFormat) {\r\n\t\t\tif (thirdPartySetsByFormat[source][format.id] && Object.keys(thirdPartySetsByFormat[source][format.id]).length) {\r\n\t\t\t\tdata[format.id] = data[format.id] || {};\r\n\t\t\t\tdata[format.id][source] = thirdPartySetsByFormat[source][format.id];\r\n\t\t\t}\r\n\t\t}\r\n\r\n\t\tconst stats = await getStatisticsURL(index, format);\r\n\t\tif (!stats) continue;\r\n\t\ttry {\r\n\t\t\tconst statistics = smogon.Statistics.process(await request(stats.url));\r\n\t\t\tconst sets = importUsageBasedSets(gen, format, statistics, stats.count);\r\n\t\t\tif (Object.keys(sets).length) {\r\n\t\t\t\tdata[format.id] = data[format.id] || {};\r\n\t\t\t\tdata[format.id]['stats'] = sets;\r\n\t\t\t}\r\n\t\t\tdata[format.id] = data[format.id] || {};\r\n\t\t} catch (err) {\r\n\t\t\terror(`${stats.url} = ${err}`);\r\n\t\t}\r\n\t}\r\n\r\n\treturn data;\r\n}\r\n\r\nfunction eligible(dex: ModdedDex, id: ID) {\r\n\tconst gen = toGen(dex, id);\r\n\tif (!gen || gen > dex.gen) return false;\r\n\r\n\tconst species = dex.species.get(id);\r\n\tif (['Mega', 'Primal', 'Ultra'].some(f => species.forme.startsWith(f))) return true;\r\n\r\n\t// Species with formes distinct enough to merit inclusion\r\n\tconst unique = ['darmanitan', 'meloetta', 'greninja', 'zygarde'];\r\n\t// Too similar to their base forme/species to matter\r\n\tconst similar = ['pichu', 'pikachu', 'genesect', 'basculin', 'magearna', 'keldeo', 'vivillon'];\r\n\r\n\tif (species.battleOnly && !unique.some(f => id.startsWith(f))) return false;\r\n\r\n\t// Most of these don't have analyses\r\n\tconst capNFE = species.isNonstandard === 'CAP' && speLine truncated
"mappings": ";;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,WAAsB;AACtB,YAAuB;AACvB,UAAqB;AACrB,WAAsB;AAEtB,aAAwB;AAExB,iBAAsB;AACtB,iBAAwB;AACxB,4BAA4B;AAC5B,eAAI,eAAe;AAgCZ,MAAM,QAAQ,oBAAI,IAAI;AAAA,EAC5B;AAAA,EAAS;AAAA,EAAM;AAAA,EAAM;AAAA,EAAM;AAAA,EAAM;AAAA,EAAM;AAAA,EAAM;AAAA,EAAM;AAAA,EAAO;AAAA,EAC1D;AAAA,EAAa;AAAA,EAAqB;AAAA,EAAqB;AAAA,EACvD;AAAA,EAAW;AAAA,EAAW;AAAA,EAAW;AAAA,EAAsB;AAAA,EAAW;AAAA,EAClE;AAAA,EAAgB;AAAA,EAAiB;AAAA,EAAoB;AAAA,EAAY;AAAA,EACjE;AACD,CAAC;AACD,MAAM,UAAU,oBAAI,IAA8C;AAClE,MAAM,aAAa,oBAAI,IAAuB;AAC9C,SAAS,MAAM,GAAG,OAAO,GAAG,OAAO;AAClC,aAAW,QAAQ,OAAO;AACzB,UAAM,SAAS,eAAI,QAAQ,IAAI,MAAM,MAAM,MAAM;AACjD,QAAI,OAAO,QAAQ;AAClB,cAAQ,IAAI,OAAO,IAAI,EAAC,KAA2B,OAAM,CAAC;AAC1D,iBAAW,IAAI,OAAO,IAAI,IAAI,oCAAc,MAAM,CAAC;AAAA,IACpD;AAAA,EACD;AACD;AAEA,eAAsB,YAAY;AACjC,QAAM,QAAQ,MAAM,QAAQ,OAAO,WAAW,GAAG;AAEjD,QAAM,UAAU,CAAC;AACjB,WAAS,MAAM,GAAG,OAAO,GAAG,OAAO;AAClC,YAAQ,KAAK,UAAU,KAAsB,KAAK,CAAC;AAAA,EACpD;AAEA,SAAO,QAAQ,IAAI,OAAO;AAC3B;AAEA,eAAe,UAAU,KAAoB,OAAe;AAC3D,QAAM,OAAuB,CAAC;AAE9B,QAAM,qBAAwD,CAAC;AAC/D,QAAM,yBAAgF,CAAC;AAEvF,QAAM,cAA4C,CAAC;AACnD,QAAM,UAAU,CAAC;AACjB,QAAM,MAAM,eAAI,UAAU,MAAM,OAAO;AACvC,aAAW,MAAM,IAAI,KAAK,SAAS;AAClC,QAAI,CAAC,SAAS,KAAK,EAAQ;AAAG;AAC9B,UAAM,UAAU,IAAI,QAAQ,IAAI,EAAE;AAClC,QAAI,QAAQ;AAAY;AACxB,YAAQ,KAAK,iBAAiB,IAAI,QAAQ,IAAI,EAAE,EAAE,MAAM,KAAK,oBAAoB,WAAW,CAAC;AAAA,EAC9F;AACA,QAAM,QAAQ,IAAI,OAAO;AAEzB,aAAW,EAAC,QAAQ,KAAK,EAAC,KAAK,QAAQ,OAAO,GAAG;AAChD,QAAI,MAAM;AAAK;AAEf,QAAI,mBAAmB,OAAO,EAAE,KAAK,OAAO,KAAK,mBAAmB,OAAO,EAAE,CAAC,EAAE,QAAQ;AACvF,WAAK,OAAO,EAAE,IAAI,CAAC;AACnB,WAAK,OAAO,EAAE,EAAE,KAAK,IAAI,mBAAmB,OAAO,EAAE;AACrD,aAAO,QAAQ,YAAY,OAAO,EAAE,GAAG,KAAK;AAAA,IAC7C;AAEA,eAAW,UAAU,wBAAwB;AAC5C,UAAI,uBAAuB,MAAM,EAAE,OAAO,EAAE,KAAK,OAAO,KAAK,uBAAuB,MAAM,EAAE,OAAO,EAAE,CAAC,EAAE,QAAQ;AAC/G,aAAK,OAAO,EAAE,IAAI,KAAK,OAAO,EAAE,KAAK,CAAC;AACtC,aAAK,OAAO,EAAE,EAAE,MAAM,IAAI,uBAAuB,MAAM,EAAE,OAAO,EAAE;AAAA,MACnE;AAAA,IACD;AAEA,UAAM,QAAQ,MAAM,iBAAiB,OAAO,MAAM;AAClD,QAAI,CAAC;AAAO;AACZ,QAAI;AACH,YAAM,aAAa,OAAO,WAAW,QAAQ,MAAM,QAAQ,MAAM,GAAG,CAAC;AACrE,YAAM,OAAO,qBAAqB,KAAK,QAAQ,YAAY,MAAM,KAAK;AACtE,UAAI,OAAO,KAAK,IAAI,EAAE,QAAQ;AAC7B,aAAK,OAAO,EAAE,IAAI,KAAK,OAAO,EAAE,KAAK,CAAC;AACtC,aAAK,OAAO,EAAE,EAAE,OAAO,IAAI;AAAA,MAC5B;AACA,WAAK,OAAO,EAAE,IAAI,KAAK,OAAO,EAAE,KAAK,CAAC;AAAA,IACvC,SAAS,KAAP;AACD,YAAM,GAAG,MAAM,SAAS,KAAK;AAAA,IAC9B;AAAA,EACD;AAEA,SAAO;AACR;AAEA,SAAS,SAAS,KAAgB,IAAQ;AACzC,QAAM,MAAM,MAAM,KAAK,EAAE;AACzB,MAAI,CAAC,OAAO,MAAM,IAAI;AAAK,WAAO;AAElC,QAAM,UAAU,IAAI,QAAQ,IAAI,EAAE;AAClC,MAAI,CAAC,QAAQ,UAAU,OAAO,EAAE,KAAK,OAAK,QAAQ,MAAM,WAAW,CAAC,CAAC;AAAG,WAAO;AAG/E,QAAM,SAAS,CAAC,cAAc,YAAY,YAAY,SAAS;AAE/D,QAAM,UAAU,CAAC,SAAS,WAAW,YAAY,YAAY,YAAY,UAAU,UAAU;AAE7F,MAAI,QAAQ,cAAc,CAAC,OAAO,KAAK,OAAK,GAAG,WAAW,CAAC,CAAC;AAAG,WAAO;AAGtE,QAAM,SAAS,QAAQ,kBAAkB,SAAS,QAAQ;AAE1D,SAAO,CAAC,GAAG,SAAS,OAAO,KAAK,CAAC,UAAU,CAAC,QAAQ,KAAK,OAAK,GAAG,WAAW,CAAC,KAAK,OAAO,CAAC;AAC3F;AAGA,SAAS,MAAM,KAAgB,MAAyC;AACvE,QAAM,UAAU,IAAI,QAAQ,IAAI,IAAI;AACpC,MAAI,QAAQ,kBAAkB;AAAQ,WAAO;AAC7C,MAAI,CAAC,QAAQ,UAAW,QAAQ,iBAAiB,QAAQ,kBAAkB;AAAQ,WAAO;AAE1F,QAAM,IAAI,QAAQ;AAClB,MAAI,IAAI;AAAK,WAAO;AACpB,MAAI,IAAI;AAAK,WAAO;AACpB,MAAI,IAAI,OAAQ,KAAK,OAAO,KAAK,OAAS,KAAK,QAAQ,KAAK;AAAO,WAAO;AAC1E,MAAI,IAAI,OAAQ,KAAK,MAAM,KAAK,OAAS,KAAK,QAAQ,KAAK;AAAO,WAAO;AACzE,MAAI,IAAI,OAAQ,KAAK,OAAO,KAAK,OAAS,KAAK,QAAQ,KAAK;AAAO,WAAO;AAC1E,MAAI,IAAI,OAAQ,KAAK,MAAM,KAAK,OAAS,KAAK,QAAQ,KAAK,QAAU,KAAK,QAAQ,KAAK;AAAO,WAAO;AACrG,MAAI,IAAI;AAAK,WAAO;AACpB,MAAI,IAAI;AAAK,WAAO;AACpB,MAAI,IAAI;AAAG,WAAO;AACnB;AAEA,eAAe,iBACd,SACA,KACA,cACA,aACC;AACD,QAAM,mBAAmB,MAAM,oBAAoB,SAAS,GAAG;AAC/D,MAAI,CAAC;AAAkB;AAEvB,aAAW,CAAC,QAAQ,QAAQ,KAAK,iBAAiB,QAAQ,GAAG;AAC5D,UAAM,MAAM,eAAI,UAAU,MAAM;AAChC,QAAI,iBAAiB,aAAa,OAAO,EAAE;AAC3C,QAAI,CAAC,gBAAgB;AACpB,uBAAiB,CAAC;AAClB,mBAAa,OAAO,EAAE,IAAI;AAAA,IAC3B;AAEA,QAAI,cAAc,IAAI,QAAQ,IAAI,OAAO;AACzC,QAAI,YAAY,gBAAgB,YAAY;AAAM,oBAAc,IAAI,QAAQ,IAAI,YAAY,WAAW;AACvG,UAAM,mBAA8B,CAAC;AACrC,QAAI,YAAY,aAAa;AAC5B,iBAAW,SAAS,YAAY,aAAa;AAC5C,cAAM,eAAe,IAAI,QAAQ,IAAI,KAAK;AAC1C,YAAI,aAAa,cAAc,SAAS,SAAK,iBAAK,YAAY,CAAC,GAAG;AACjE,2BAAiB,KAAK,YAAY;AAAA,QACnC;AAAA,MACD;AAAA,IACD;AAEA,eAAW,YAAY,UAAU;AAChC,iBAAW,WAAW,SAAS,UAAU;AACxC,cAAM,MAAM,oBAAoB,KAAK,QAAQ,SAAS,OAAO;AAC7D,cAAM,OAAO,UAAU,QAAQ,IAAI;AACnC,qBAAa,KAAK,QAAQ,SAAS,MAAM,KAAK,gBAAgB,WAAW;AACzE,mBAAW,mBAAmB,kBAAkB;AAE/C,gBAAM,IAAI,EAAC,GAAG,IAAG;AACjB,cAAI,CAAC,OAAO,GAAG,SAAS,kBAAkB;AAAG,cAAE,UAAU,gBAAgB,UAAU,CAAC;AACpF,cAAI,OAAO,gBAAgB,eAAe,UAAU;AACnD,gBAAI,CAAC,gBAAgB,WAAY,SAAS,OAAO;AAAG;AACpD,kBAAM,UAAU,IAAI,QAAQ,IAAI,OAAO;AACvC,kBAAM,gBAAgB,GAAG,UAAU,QAAQ,aAAa,QAAQ;AAChE,yBAAa,KAAK,QAAQ,gBAAgB,MAAM,eAAe,GAAG,gBAAgB,aAAa,OAAO;AAAA,UACvG,WAAW,gBAAgB,eAAe,SAAS;AAClD,yBAAa,KAAK,QAAQ,gBAAgB,MAAM,MAAM,GAAG,gBAAgB,WAAW;AAAA,UACrF;AAAA,QACD;AAAA,MACD;AAAA,IACD;AAAA,EACD;AACD;AAEA,SAAS,aACR,KACA,QACA,SACA,MACA,KACA,gBACA,aACA,wBACC;AACD,MAAI,SAAS,OAAO,KAAK,QAAQ,SAAS,MAAM,KAAK,sBAAsB,GAAG;AAC7E,mBAAe,OAAO,IAAI,eAAe,OAAO,KAAK,CAAC;AACtD,mBAAe,OAAO,EAAE,IAAI,IAAI;AAChC,gBAAY,OAAO,EAAE,KAAK,YAAY,OAAO,EAAE,KAAK,Line truncated
"names": []
}