Initial Modpack Upload

This commit is contained in:
Rbanh committed 2023-08-14 21:45:09 -04:00
1 parent 268b0650aa
commit 3d8b956ee4
1429 files changed
+820128

No files matched your search

+32
View File
@@ -0,0 +1,32 @@
"use strict";
var __defProp = Object.defineProperty;
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
var __getOwnPropNames = Object.getOwnPropertyNames;
var __hasOwnProp = Object.prototype.hasOwnProperty;
var __export = (target, all) => {
for (var name in all)
__defProp(target, name, { get: all[name], enumerable: true });
};
var __copyProps = (to, from, except, desc) => {
if (from && typeof from === "object" || typeof from === "function") {
for (let key of __getOwnPropNames(from))
if (!__hasOwnProp.call(to, key) && key !== except)
__defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
}
return to;
};
var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
var artemis_exports = {};
__export(artemis_exports, {
LocalClassifier: () => import_local.LocalClassifier,
RemoteClassifier: () => import_remote.RemoteClassifier,
destroy: () => destroy
});
module.exports = __toCommonJS(artemis_exports);
var import_local = require("./local");
var import_remote = require("./remote");
function destroy() {
void import_local.LocalClassifier.destroy();
void import_remote.RemoteClassifier.PM.destroy();
}
//# sourceMappingURL=index.js.map
+7
View File
@@ -0,0 +1,7 @@
{
"version": 3,
"sources": ["../../../../server/artemis/index.ts"],
"sourcesContent": ["/**\r\n * @author mia-pi-git\r\n */\r\nimport {LocalClassifier} from './local';\r\nimport {RemoteClassifier} from './remote';\r\n\r\nexport {LocalClassifier, RemoteClassifier};\r\n\r\nexport function destroy() {\r\n\tvoid LocalClassifier.destroy();\r\n\tvoid RemoteClassifier.PM.destroy();\r\n}\r\n"],
"mappings": ";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAGA,mBAA8B;AAC9B,oBAA+B;AAIxB,SAAS,UAAU;AACzB,OAAK,6BAAgB,QAAQ;AAC7B,OAAK,+BAAiB,GAAG,QAAQ;AAClC;",
"names": []
}
+207
View File
@@ -0,0 +1,207 @@
"use strict";
var __create = Object.create;
var __defProp = Object.defineProperty;
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
var __getOwnPropNames = Object.getOwnPropertyNames;
var __getProtoOf = Object.getPrototypeOf;
var __hasOwnProp = Object.prototype.hasOwnProperty;
var __export = (target, all) => {
for (var name in all)
__defProp(target, name, { get: all[name], enumerable: true });
};
var __copyProps = (to, from, except, desc) => {
if (from && typeof from === "object" || typeof from === "function") {
for (let key of __getOwnPropNames(from))
if (!__hasOwnProp.call(to, key) && key !== except)
__defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
}
return to;
};
var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
// If the importer is in node compatibility mode or this is not an ESM
// file that has been converted to a CommonJS file using a Babel-
// compatible transform (i.e. "__esModule" has not been set), then set
// "default" to the CommonJS "module.exports" for node compatibility.
isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
mod
));
var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
var local_exports = {};
__export(local_exports, {
LocalClassifier: () => LocalClassifier,
PM: () => PM
});
module.exports = __toCommonJS(local_exports);
var child_process = __toESM(require("child_process"));
var import_lib = require("../../lib");
var import_config_loader = require("../config-loader");
var import_dex_data = require("../../sim/dex-data");
class ArtemisStream extends import_lib.Streams.ObjectReadWriteStream {
constructor() {
super();
this.tasks = /* @__PURE__ */ new Set();
this.process = child_process.spawn("python3", [
"-u",
(0, import_lib.FS)("server/artemis/model.py").path,
import_config_loader.Config.debugartemisprocesses ? "debug" : ""
].filter(Boolean));
this.listen();
}
listen() {
this.process.stdout.setEncoding("utf8");
this.process.stderr.setEncoding("utf8");
this.process.stdout.on("data", (data) => {
data = data.trim();
const [taskId, dataStr] = data.split("|");
if (this.tasks.has(taskId)) {
this.tasks.delete(taskId);
return this.push(`${taskId}
${dataStr}`);
}
if (taskId === "error") {
const info = JSON.parse(dataStr);
Monitor.crashlog(new Error(info.error), "An Artemis script", info);
try {
this.pushEnd();
this.process.disconnect();
} catch {
}
}
});
this.process.stderr.on("data", (data) => {
if (/Downloading: ([0-9]+)%/i.test(data)) {
return;
}
Monitor.crashlog(new Error(data), "An Artemis process");
});
this.process.on("error", (err) => {
Monitor.crashlog(err, "An Artemis process");
this.pushEnd();
});
this.process.on("close", () => {
this.pushEnd();
});
}
_write(chunk) {
const [taskId, message] = import_lib.Utils.splitFirst(chunk, "\n");
this.tasks.add(taskId);
this.process.stdin.write(`${taskId}|${message}
`);
}
destroy() {
try {
this.process.kill();
} catch {
}
this.pushEnd();
}
}
const PM = new import_lib.ProcessManager.StreamProcessManager(module, () => new ArtemisStream(), (message) => {
if (message.startsWith("SLOW\n")) {
Monitor.slow(message.slice(5));
}
});
const _LocalClassifier = class {
constructor() {
this.enabled = false;
this.requests = /* @__PURE__ */ new Map();
this.lastTask = 0;
this.readyPromise = null;
_LocalClassifier.classifiers.push(this);
void this.setupProcesses();
}
static destroy() {
for (const classifier of this.classifiers)
void classifier.destroy();
return this.PM.destroy();
}
async setupProcesses() {
this.readyPromise = new Promise((resolve) => {
child_process.exec('python3 -c "import detoxify"', (err, out, stderr) => {
if (err || stderr) {
resolve(false);
} else {
resolve(true);
}
});
});
const res = await this.readyPromise;
this.enabled = res;
this.readyPromise = null;
if (res) {
this.stream = PM.createStream();
void this.listen();
}
}
async listen() {
if (!this.stream)
return null;
for await (const chunk of this.stream) {
const [rawTaskId, data] = import_lib.Utils.splitFirst(chunk, "\n");
const task = parseInt(rawTaskId);
const resolver = this.requests.get(task);
if (resolver) {
resolver(JSON.parse(data));
this.requests.delete(task);
}
}
}
destroy() {
_LocalClassifier.classifiers.splice(_LocalClassifier.classifiers.indexOf(this), 1);
return this.stream?.destroy();
}
async classify(text) {
if (this.readyPromise)
await this.readyPromise;
if (!this.stream)
return null;
const taskId = this.lastTask++;
const data = await new Promise((resolve) => {
this.requests.set(taskId, resolve);
void this.stream?.write(`${taskId}
${text}`);
});
for (const k in data) {
data[k] = parseFloat(data[k]);
}
return data;
}
};
let LocalClassifier = _LocalClassifier;
LocalClassifier.PM = PM;
LocalClassifier.ATTRIBUTES = {
sexual_explicit: {},
severe_toxicity: {},
toxicity: {},
obscene: {},
identity_attack: {},
insult: {},
threat: {}
};
LocalClassifier.classifiers = [];
if (require.main === module) {
global.Config = import_config_loader.Config;
global.Monitor = {
crashlog(error, source = "A local Artemis child process", details = null) {
const repr = JSON.stringify([error.name, error.message, source, details]);
process.send(`THROW
@!!@${repr}
${error.stack}`);
},
slow(text) {
process.send(`CALLBACK
SLOW
${text}`);
}
};
global.toID = import_dex_data.toID;
process.on("uncaughtException", (err) => {
if (import_config_loader.Config.crashguard) {
Monitor.crashlog(err, "A local Artemis child process");
}
});
import_lib.Repl.start(`abusemonitor-local-${process.pid}`, (cmd) => eval(cmd));
} else if (!process.send) {
PM.spawn(import_config_loader.Config.localartemisprocesses || 1);
}
//# sourceMappingURL=local.js.map
+7
View File
@@ -0,0 +1,7 @@
{
"version": 3,
"sources": ["../../../../server/artemis/local.ts"],
"sourcesContent": ["/**\r\n * Typescript wrapper around the Python Artemis model.\r\n * By Mia.\r\n * @author mia-pi-git\r\n */\r\n\r\nimport * as child_process from 'child_process';\r\nimport {ProcessManager, Streams, Utils, Repl, FS} from '../../lib';\r\nimport {Config} from '../config-loader';\r\nimport {toID} from '../../sim/dex-data';\r\n\r\nclass ArtemisStream extends Streams.ObjectReadWriteStream<string> {\r\n\ttasks = new Set<string>();\r\n\tprivate process: child_process.ChildProcessWithoutNullStreams;\r\n\tconstructor() {\r\n\t\tsuper();\r\n\t\tthis.process = child_process.spawn('python3', [\r\n\t\t\t'-u', FS('server/artemis/model.py').path, Config.debugartemisprocesses ? \"debug\" : \"\",\r\n\t\t].filter(Boolean));\r\n\t\tthis.listen();\r\n\t}\r\n\tlisten() {\r\n\t\tthis.process.stdout.setEncoding('utf8');\r\n\t\tthis.process.stderr.setEncoding('utf8');\r\n\t\tthis.process.stdout.on('data', (data) => {\r\n\t\t\t// so many bugs were created by \\nready\\n\r\n\t\t\tdata = data.trim();\r\n\t\t\tconst [taskId, dataStr] = data.split(\"|\");\r\n\t\t\tif (this.tasks.has(taskId)) {\r\n\t\t\t\tthis.tasks.delete(taskId);\r\n\t\t\t\treturn this.push(`${taskId}\\n${dataStr}`);\r\n\t\t\t}\r\n\t\t\tif (taskId === 'error') { // there was a major crash and the script is no longer running\r\n\t\t\t\tconst info = JSON.parse(dataStr);\r\n\t\t\t\tMonitor.crashlog(new Error(info.error), \"An Artemis script\", info);\r\n\t\t\t\ttry {\r\n\t\t\t\t\tthis.pushEnd(); // push end first so the stream always closes\r\n\t\t\t\t\tthis.process.disconnect();\r\n\t\t\t\t} catch {}\r\n\t\t\t}\r\n\t\t});\r\n\t\tthis.process.stderr.on('data', data => {\r\n\t\t\tif (/Downloading: ([0-9]+)%/i.test(data)) {\r\n\t\t\t\t// this prints to stderr fsr and it should not be throwing\r\n\t\t\t\treturn;\r\n\t\t\t}\r\n\t\t\tMonitor.crashlog(new Error(data), \"An Artemis process\");\r\n\t\t});\r\n\t\tthis.process.on('error', err => {\r\n\t\t\tMonitor.crashlog(err, \"An Artemis process\");\r\n\t\t\tthis.pushEnd();\r\n\t\t});\r\n\t\tthis.process.on('close', () => {\r\n\t\t\tthis.pushEnd();\r\n\t\t});\r\n\t}\r\n\t_write(chunk: string) {\r\n\t\tconst [taskId, message] = Utils.splitFirst(chunk, '\\n');\r\n\t\tthis.tasks.add(taskId);\r\n\t\tthis.process.stdin.write(`${taskId}|${message}\\n`);\r\n\t}\r\n\tdestroy() {\r\n\t\ttry {\r\n\t\t\tthis.process.kill();\r\n\t\t} catch {}\r\n\t\tthis.pushEnd();\r\n\t}\r\n}\r\n\r\nexport const PM = new ProcessManager.StreamProcessManager(module, () => new ArtemisStream(), message => {\r\n\tif (message.startsWith('SLOW\\n')) {\r\n\t\tMonitor.slow(message.slice(5));\r\n\t}\r\n});\r\n\r\nexport class LocalClassifier {\r\n\tstatic readonly PM = PM;\r\n\tstatic readonly ATTRIBUTES: Record<string, unknown> = {\r\n\t\tsexual_explicit: {},\r\n\t\tsevere_toxicity: {},\r\n\t\ttoxicity: {},\r\n\t\tobscene: {},\r\n\t\tidentity_attack: {},\r\n\t\tinsult: {},\r\n\t\tthreat: {},\r\n\t};\r\n\tstatic classifiers: LocalClassifier[] = [];\r\n\tstatic destroy() {\r\n\t\tfor (const classifier of this.classifiers) void classifier.destroy();\r\n\t\treturn this.PM.destroy();\r\n\t}\r\n\t/** If stream exists, model is usable */\r\n\tstream?: Streams.ObjectReadWriteStream<string>;\r\n\tenabled = false;\r\n\trequests = new Map<number, (data: any) => void>();\r\n\tlastTask = 0;\r\n\treadyPromise: Promise<boolean> | null = null;\r\n\tconstructor() {\r\n\t\tLocalClassifier.classifiers.push(this);\r\n\t\tvoid this.setupProcesses();\r\n\t}\r\n\tasync setupProcesses() {\r\n\t\tthis.readyPromise = new Promise(resolve => {\r\n\t\t\tchild_process.exec('python3 -c \"import detoxify\"', (err, out, stderr) => {\r\n\t\t\t\tif (err || stderr) {\r\n\t\t\t\t\tresolve(false);\r\n\t\t\t\t} else {\r\n\t\t\t\t\tresolve(true);\r\n\t\t\t\t}\r\n\t\t\t});\r\n\t\t});\r\n\t\tconst res = await this.readyPromise;\r\n\t\tthis.enabled = res;\r\n\t\tthis.readyPromise = null;\r\n\t\tif (res) {\r\n\t\t\tthis.stream = PM.createStream();\r\n\t\t\tvoid this.listen();\r\n\t\t}\r\n\t}\r\n\tasync listen() {\r\n\t\tif (!this.stream) return null;\r\n\t\tfor await (const chunk of this.stream) {\r\n\t\t\tconst [rawTaskId, data] = Utils.splitFirst(chunk, '\\n');\r\n\t\t\tconst task = parseInt(rawTaskId);\r\n\t\t\tconst resolver = this.requests.get(task);\r\n\t\t\tif (resolver) {\r\n\t\t\t\tresolver(JSON.parse(data));\r\n\t\t\t\tthis.requests.delete(task);\r\n\t\t\t}\r\n\t\t}\r\n\t}\r\n\tdestroy() {\r\n\t\tLocalClassifier.classifiers.splice(LocalClassifier.classifiers.indexOf(this), 1);\r\n\t\treturn this.stream?.destroy();\r\n\t}\r\n\tasync classify(text: string) {\r\n\t\tif (this.readyPromise) await this.readyPromise;\r\n\t\tif (!this.stream) return null;\r\n\t\tconst taskId = this.lastTask++;\r\n\t\tconst data = await new Promise<any>(resolve => {\r\n\t\t\tthis.requests.set(taskId, resolve);\r\n\t\t\tvoid this.stream?.write(`${taskId}\\n${text}`);\r\n\t\t});\r\n\t\tfor (const k in data) {\r\n\t\t\t// floats have to be made into strings because python's json.dumps\r\n\t\t\t// doesn'Line truncated
"mappings": ";;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAMA,oBAA+B;AAC/B,iBAAuD;AACvD,2BAAqB;AACrB,sBAAmB;AAEnB,MAAM,sBAAsB,mBAAQ,sBAA8B;AAAA,EAGjE,cAAc;AACb,UAAM;AAHP,iBAAQ,oBAAI,IAAY;AAIvB,SAAK,UAAU,cAAc,MAAM,WAAW;AAAA,MAC7C;AAAA,UAAM,eAAG,yBAAyB,EAAE;AAAA,MAAM,4BAAO,wBAAwB,UAAU;AAAA,IACpF,EAAE,OAAO,OAAO,CAAC;AACjB,SAAK,OAAO;AAAA,EACb;AAAA,EACA,SAAS;AACR,SAAK,QAAQ,OAAO,YAAY,MAAM;AACtC,SAAK,QAAQ,OAAO,YAAY,MAAM;AACtC,SAAK,QAAQ,OAAO,GAAG,QAAQ,CAAC,SAAS;AAExC,aAAO,KAAK,KAAK;AACjB,YAAM,CAAC,QAAQ,OAAO,IAAI,KAAK,MAAM,GAAG;AACxC,UAAI,KAAK,MAAM,IAAI,MAAM,GAAG;AAC3B,aAAK,MAAM,OAAO,MAAM;AACxB,eAAO,KAAK,KAAK,GAAG;AAAA,EAAW,SAAS;AAAA,MACzC;AACA,UAAI,WAAW,SAAS;AACvB,cAAM,OAAO,KAAK,MAAM,OAAO;AAC/B,gBAAQ,SAAS,IAAI,MAAM,KAAK,KAAK,GAAG,qBAAqB,IAAI;AACjE,YAAI;AACH,eAAK,QAAQ;AACb,eAAK,QAAQ,WAAW;AAAA,QACzB,QAAE;AAAA,QAAO;AAAA,MACV;AAAA,IACD,CAAC;AACD,SAAK,QAAQ,OAAO,GAAG,QAAQ,UAAQ;AACtC,UAAI,0BAA0B,KAAK,IAAI,GAAG;AAEzC;AAAA,MACD;AACA,cAAQ,SAAS,IAAI,MAAM,IAAI,GAAG,oBAAoB;AAAA,IACvD,CAAC;AACD,SAAK,QAAQ,GAAG,SAAS,SAAO;AAC/B,cAAQ,SAAS,KAAK,oBAAoB;AAC1C,WAAK,QAAQ;AAAA,IACd,CAAC;AACD,SAAK,QAAQ,GAAG,SAAS,MAAM;AAC9B,WAAK,QAAQ;AAAA,IACd,CAAC;AAAA,EACF;AAAA,EACA,OAAO,OAAe;AACrB,UAAM,CAAC,QAAQ,OAAO,IAAI,iBAAM,WAAW,OAAO,IAAI;AACtD,SAAK,MAAM,IAAI,MAAM;AACrB,SAAK,QAAQ,MAAM,MAAM,GAAG,UAAU;AAAA,CAAW;AAAA,EAClD;AAAA,EACA,UAAU;AACT,QAAI;AACH,WAAK,QAAQ,KAAK;AAAA,IACnB,QAAE;AAAA,IAAO;AACT,SAAK,QAAQ;AAAA,EACd;AACD;AAEO,MAAM,KAAK,IAAI,0BAAe,qBAAqB,QAAQ,MAAM,IAAI,cAAc,GAAG,aAAW;AACvG,MAAI,QAAQ,WAAW,QAAQ,GAAG;AACjC,YAAQ,KAAK,QAAQ,MAAM,CAAC,CAAC;AAAA,EAC9B;AACD,CAAC;AAEM,MAAM,mBAAN,MAAsB;AAAA,EAsB5B,cAAc;AAJd,mBAAU;AACV,oBAAW,oBAAI,IAAiC;AAChD,oBAAW;AACX,wBAAwC;AAEvC,qBAAgB,YAAY,KAAK,IAAI;AACrC,SAAK,KAAK,eAAe;AAAA,EAC1B;AAAA,EAbA,OAAO,UAAU;AAChB,eAAW,cAAc,KAAK;AAAa,WAAK,WAAW,QAAQ;AACnE,WAAO,KAAK,GAAG,QAAQ;AAAA,EACxB;AAAA,EAWA,MAAM,iBAAiB;AACtB,SAAK,eAAe,IAAI,QAAQ,aAAW;AAC1C,oBAAc,KAAK,gCAAgC,CAAC,KAAK,KAAK,WAAW;AACxE,YAAI,OAAO,QAAQ;AAClB,kBAAQ,KAAK;AAAA,QACd,OAAO;AACN,kBAAQ,IAAI;AAAA,QACb;AAAA,MACD,CAAC;AAAA,IACF,CAAC;AACD,UAAM,MAAM,MAAM,KAAK;AACvB,SAAK,UAAU;AACf,SAAK,eAAe;AACpB,QAAI,KAAK;AACR,WAAK,SAAS,GAAG,aAAa;AAC9B,WAAK,KAAK,OAAO;AAAA,IAClB;AAAA,EACD;AAAA,EACA,MAAM,SAAS;AACd,QAAI,CAAC,KAAK;AAAQ,aAAO;AACzB,qBAAiB,SAAS,KAAK,QAAQ;AACtC,YAAM,CAAC,WAAW,IAAI,IAAI,iBAAM,WAAW,OAAO,IAAI;AACtD,YAAM,OAAO,SAAS,SAAS;AAC/B,YAAM,WAAW,KAAK,SAAS,IAAI,IAAI;AACvC,UAAI,UAAU;AACb,iBAAS,KAAK,MAAM,IAAI,CAAC;AACzB,aAAK,SAAS,OAAO,IAAI;AAAA,MAC1B;AAAA,IACD;AAAA,EACD;AAAA,EACA,UAAU;AACT,qBAAgB,YAAY,OAAO,iBAAgB,YAAY,QAAQ,IAAI,GAAG,CAAC;AAC/E,WAAO,KAAK,QAAQ,QAAQ;AAAA,EAC7B;AAAA,EACA,MAAM,SAAS,MAAc;AAC5B,QAAI,KAAK;AAAc,YAAM,KAAK;AAClC,QAAI,CAAC,KAAK;AAAQ,aAAO;AACzB,UAAM,SAAS,KAAK;AACpB,UAAM,OAAO,MAAM,IAAI,QAAa,aAAW;AAC9C,WAAK,SAAS,IAAI,QAAQ,OAAO;AACjC,WAAK,KAAK,QAAQ,MAAM,GAAG;AAAA,EAAW,MAAM;AAAA,IAC7C,CAAC;AACD,eAAW,KAAK,MAAM;AAGrB,WAAK,CAAC,IAAI,WAAW,KAAK,CAAC,CAAC;AAAA,IAC7B;AACA,WAAO;AAAA,EACR;AACD;AA3EO,IAAM,kBAAN;AAAM,gBACI,KAAK;AADT,gBAEI,aAAsC;AAAA,EACrD,iBAAiB,CAAC;AAAA,EAClB,iBAAiB,CAAC;AAAA,EAClB,UAAU,CAAC;AAAA,EACX,SAAS,CAAC;AAAA,EACV,iBAAiB,CAAC;AAAA,EAClB,QAAQ,CAAC;AAAA,EACT,QAAQ,CAAC;AACV;AAVY,gBAWL,cAAiC,CAAC;AAoE1C,IAAI,QAAQ,SAAS,QAAQ;AAE5B,SAAO,SAAS;AAChB,SAAO,UAAU;AAAA,IAChB,SAAS,OAAc,SAAS,iCAAiC,UAA4B,MAAM;AAClG,YAAM,OAAO,KAAK,UAAU,CAAC,MAAM,MAAM,MAAM,SAAS,QAAQ,OAAO,CAAC;AACxE,cAAQ,KAAM;AAAA,MAAc;AAAA,EAAS,MAAM,OAAO;AAAA,IACnD;AAAA,IACA,KAAK,MAAc;AAClB,cAAQ,KAAM;AAAA;AAAA,EAAmB,MAAM;AAAA,IACxC;AAAA,EACD;AACA,SAAO,OAAO;AACd,UAAQ,GAAG,qBAAqB,SAAO;AACtC,QAAI,4BAAO,YAAY;AACtB,cAAQ,SAAS,KAAK,+BAA+B;AAAA,IACtD;AAAA,EACD,CAAC;AAED,kBAAK,MAAM,sBAAsB,QAAQ,OAAO,SAAO,KAAK,GAAG,CAAC;AACjE,WAAW,CAAC,QAAQ,MAAM;AACzB,KAAG,MAAM,4BAAO,yBAAyB,CAAC;AAC3C;",
"names": []
}
+186
View File
@@ -0,0 +1,186 @@
"use strict";
var __defProp = Object.defineProperty;
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
var __getOwnPropNames = Object.getOwnPropertyNames;
var __hasOwnProp = Object.prototype.hasOwnProperty;
var __export = (target, all) => {
for (var name in all)
__defProp(target, name, { get: all[name], enumerable: true });
};
var __copyProps = (to, from, except, desc) => {
if (from && typeof from === "object" || typeof from === "function") {
for (let key of __getOwnPropNames(from))
if (!__hasOwnProp.call(to, key) && key !== except)
__defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
}
return to;
};
var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
var remote_exports = {};
__export(remote_exports, {
ATTRIBUTES: () => ATTRIBUTES,
Limiter: () => Limiter,
PM: () => PM,
RemoteClassifier: () => RemoteClassifier,
limiter: () => limiter
});
module.exports = __toCommonJS(remote_exports);
var import_lib = require("../../lib");
var import_config_loader = require("../config-loader");
var import_dex_data = require("../../sim/dex-data");
const PM_TIMEOUT = 20 * 60 * 1e3;
const ATTRIBUTES = {
"SEVERE_TOXICITY": {},
"TOXICITY": {},
"IDENTITY_ATTACK": {},
"INSULT": {},
"PROFANITY": {},
"THREAT": {},
"SEXUALLY_EXPLICIT": {},
"FLIRTATION": {}
};
function time() {
return Math.floor(Math.floor(Date.now() / 1e3) / 60);
}
class Limiter {
constructor(max) {
this.lastTick = time();
this.count = 0;
this.max = max;
}
shouldRequest() {
const now = time();
if (this.lastTick !== now) {
this.count = 0;
this.lastTick = now;
}
this.count++;
return this.count < this.max;
}
}
function isCommon(message) {
message = message.toLowerCase().replace(/\?!\., ;:/g, "");
return ["gg", "wp", "ggwp", "gl", "hf", "glhf", "hello"].includes(message);
}
let throttleTime = null;
const limiter = new Limiter(800);
const PM = new import_lib.ProcessManager.QueryProcessManager(module, async (text) => {
if (isCommon(text) || !limiter.shouldRequest())
return null;
if (throttleTime && Date.now() - throttleTime < 1e4) {
return null;
}
if (throttleTime)
throttleTime = null;
const requestData = {
// todo - support 'es', 'it', 'pt', 'fr' - use user.language? room.settings.language...?
languages: ["en"],
requestedAttributes: ATTRIBUTES,
comment: { text }
};
try {
const raw = await (0, import_lib.Net)(`https://commentanalyzer.googleapis.com/v1alpha1/comments:analyze`).post({
query: {
key: import_config_loader.Config.perspectiveKey
},
body: JSON.stringify(requestData),
headers: {
"Content-Type": "application/json"
},
timeout: 10 * 1e3
// 10s
});
if (!raw)
return null;
const data = JSON.parse(raw);
if (data.error)
throw new Error(data.message);
const result = {};
for (const k in data.attributeScores) {
const score = data.attributeScores[k];
result[k] = score.summaryScore.value;
}
return result;
} catch (e) {
throttleTime = Date.now();
if (e.message.startsWith("Request timeout")) {
return null;
}
Monitor.crashlog(e, "A Perspective API request", { request: JSON.stringify(requestData) });
return null;
}
}, PM_TIMEOUT);
if (require.main === module) {
global.Config = import_config_loader.Config;
global.Monitor = {
crashlog(error, source = "A remote Artemis child process", details = null) {
const repr = JSON.stringify([error.name, error.message, source, details]);
process.send(`THROW
@!!@${repr}
${error.stack}`);
},
slow(text) {
process.send(`CALLBACK
SLOW
${text}`);
}
};
global.toID = import_dex_data.toID;
process.on("uncaughtException", (err) => {
if (import_config_loader.Config.crashguard) {
Monitor.crashlog(err, "A remote Artemis child process");
}
});
import_lib.Repl.start(`abusemonitor-remote-${process.pid}`, (cmd) => eval(cmd));
} else if (!process.send) {
PM.spawn(import_config_loader.Config.remoteartemisprocesses || 1);
}
class RemoteClassifier {
classify(text) {
if (!import_config_loader.Config.perspectiveKey)
return Promise.resolve(null);
return PM.query(text);
}
async suggestScore(text, data) {
if (!import_config_loader.Config.perspectiveKey)
return Promise.resolve(null);
const body = {
comment: { text },
attributeScores: {}
};
for (const k in data) {
body.attributeScores[k] = { summaryScore: { value: data[k] } };
}
try {
const raw = await (0, import_lib.Net)(`https://commentanalyzer.googleapis.com/v1alpha1/comments:suggestscore`).post({
query: {
key: import_config_loader.Config.perspectiveKey
},
body: JSON.stringify(body),
headers: {
"Content-Type": "application/json"
},
timeout: 10 * 1e3
// 10s
});
return JSON.parse(raw);
} catch (e) {
return { error: e.message };
}
}
destroy() {
return PM.destroy();
}
respawn() {
return PM.respawn();
}
spawn(number) {
PM.spawn(number);
}
getActiveProcesses() {
return PM.processes.length;
}
}
RemoteClassifier.PM = PM;
RemoteClassifier.ATTRIBUTES = ATTRIBUTES;
//# sourceMappingURL=remote.js.map
+7
View File
@@ -0,0 +1,7 @@
{
"version": 3,
"sources": ["../../../../server/artemis/remote.ts"],
"sourcesContent": ["/**\r\n * Code for using Google's Perspective API for filters.\r\n * @author mia-pi-git\r\n */\r\nimport {ProcessManager, Net, Repl} from '../../lib';\r\nimport {Config} from '../config-loader';\r\nimport {toID} from '../../sim/dex-data';\r\n\r\n// 20m. this is mostly here so we can use Monitor.slow()\r\nconst PM_TIMEOUT = 20 * 60 * 1000;\r\nexport const ATTRIBUTES = {\r\n\t\"SEVERE_TOXICITY\": {},\r\n\t\"TOXICITY\": {},\r\n\t\"IDENTITY_ATTACK\": {},\r\n\t\"INSULT\": {},\r\n\t\"PROFANITY\": {},\r\n\t\"THREAT\": {},\r\n\t\"SEXUALLY_EXPLICIT\": {},\r\n\t\"FLIRTATION\": {},\r\n};\r\n\r\n\r\nexport interface PerspectiveRequest {\r\n\tlanguages: string[];\r\n\trequestedAttributes: AnyObject;\r\n\tcomment: {text: string};\r\n}\r\n\r\nfunction time() {\r\n\treturn Math.floor(Math.floor(Date.now() / 1000) / 60);\r\n}\r\n\r\nexport class Limiter {\r\n\treadonly max: number;\r\n\tlastTick = time();\r\n\tcount = 0;\r\n\tconstructor(max: number) {\r\n\t\tthis.max = max;\r\n\t}\r\n\tshouldRequest() {\r\n\t\tconst now = time();\r\n\t\tif (this.lastTick !== now) {\r\n\t\t\tthis.count = 0;\r\n\t\t\tthis.lastTick = now;\r\n\t\t}\r\n\t\tthis.count++;\r\n\t\treturn this.count < this.max;\r\n\t}\r\n}\r\n\r\nfunction isCommon(message: string) {\r\n\tmessage = message.toLowerCase().replace(/\\?!\\., ;:/g, '');\r\n\treturn ['gg', 'wp', 'ggwp', 'gl', 'hf', 'glhf', 'hello'].includes(message);\r\n}\r\n\r\nlet throttleTime: number | null = null;\r\nexport const limiter = new Limiter(800);\r\nexport const PM = new ProcessManager.QueryProcessManager<string, Record<string, number> | null>(module, async text => {\r\n\tif (isCommon(text) || !limiter.shouldRequest()) return null;\r\n\tif (throttleTime && (Date.now() - throttleTime < 10000)) {\r\n\t\treturn null;\r\n\t}\r\n\tif (throttleTime) throttleTime = null;\r\n\r\n\tconst requestData: PerspectiveRequest = {\r\n\t\t// todo - support 'es', 'it', 'pt', 'fr' - use user.language? room.settings.language...?\r\n\t\tlanguages: ['en'],\r\n\t\trequestedAttributes: ATTRIBUTES,\r\n\t\tcomment: {text},\r\n\t};\r\n\ttry {\r\n\t\tconst raw = await Net(`https://commentanalyzer.googleapis.com/v1alpha1/comments:analyze`).post({\r\n\t\t\tquery: {\r\n\t\t\t\tkey: Config.perspectiveKey,\r\n\t\t\t},\r\n\t\t\tbody: JSON.stringify(requestData),\r\n\t\t\theaders: {\r\n\t\t\t\t'Content-Type': \"application/json\",\r\n\t\t\t},\r\n\t\t\ttimeout: 10 * 1000, // 10s\r\n\t\t});\r\n\t\tif (!raw) return null;\r\n\t\tconst data = JSON.parse(raw);\r\n\t\tif (data.error) throw new Error(data.message);\r\n\t\tconst result: {[k: string]: number} = {};\r\n\t\tfor (const k in data.attributeScores) {\r\n\t\t\tconst score = data.attributeScores[k];\r\n\t\t\tresult[k] = score.summaryScore.value;\r\n\t\t}\r\n\t\treturn result;\r\n\t} catch (e: any) {\r\n\t\tthrottleTime = Date.now();\r\n\t\tif (e.message.startsWith('Request timeout')) {\r\n\t\t\t// just ignore this. error on their end not ours.\r\n\t\t\t// todo maybe stop sending requests for a bit?\r\n\t\t\treturn null;\r\n\t\t}\r\n\t\tMonitor.crashlog(e, 'A Perspective API request', {request: JSON.stringify(requestData)});\r\n\t\treturn null;\r\n\t}\r\n}, PM_TIMEOUT);\r\n\r\n\r\n// main module check necessary since this gets required in other non-parent processes sometimes\r\n// when that happens we do not want to take over or set up or anything\r\nif (require.main === module) {\r\n\t// This is a child process!\r\n\tglobal.Config = Config;\r\n\tglobal.Monitor = {\r\n\t\tcrashlog(error: Error, source = 'A remote Artemis child process', details: AnyObject | null = null) {\r\n\t\t\tconst repr = JSON.stringify([error.name, error.message, source, details]);\r\n\t\t\tprocess.send!(`THROW\\n@!!@${repr}\\n${error.stack}`);\r\n\t\t},\r\n\t\tslow(text: string) {\r\n\t\t\tprocess.send!(`CALLBACK\\nSLOW\\n${text}`);\r\n\t\t},\r\n\t};\r\n\tglobal.toID = toID;\r\n\tprocess.on('uncaughtException', err => {\r\n\t\tif (Config.crashguard) {\r\n\t\t\tMonitor.crashlog(err, 'A remote Artemis child process');\r\n\t\t}\r\n\t});\r\n\t// eslint-disable-next-line no-eval\r\n\tRepl.start(`abusemonitor-remote-${process.pid}`, cmd => eval(cmd));\r\n} else if (!process.send) {\r\n\tPM.spawn(Config.remoteartemisprocesses || 1);\r\n}\r\n\r\nexport class RemoteClassifier {\r\n\tstatic readonly PM = PM;\r\n\tstatic readonly ATTRIBUTES = ATTRIBUTES;\r\n\tclassify(text: string) {\r\n\t\tif (!Config.perspectiveKey) return Promise.resolve(null);\r\n\t\treturn PM.query(text);\r\n\t}\r\n\tasync suggestScore(text: string, data: Record<string, number>) {\r\n\t\tif (!Config.perspectiveKey) return Promise.resolve(null);\r\n\t\tconst body: AnyObject = {\r\n\t\t\tcomment: {text},\r\n\t\t\tattributeScores: {},\r\n\t\t};\r\n\t\tfor (const k in data) {\r\n\t\t\tbody.attributeScores[k] = {summaryScore: {value: data[k]}};\r\n\t\t}\r\n\t\ttry {\r\n\t\t\tconst raw = await Net(`https://commentanalyzer.googleapis.com/v1alpha1/comments:suggestscore`).post({\r\n\t\t\t\tquery: {\r\n\t\t\t\t\tkey: Config.perspecLine truncated
"mappings": ";;;;;;;;;;;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAIA,iBAAwC;AACxC,2BAAqB;AACrB,sBAAmB;AAGnB,MAAM,aAAa,KAAK,KAAK;AACtB,MAAM,aAAa;AAAA,EACzB,mBAAmB,CAAC;AAAA,EACpB,YAAY,CAAC;AAAA,EACb,mBAAmB,CAAC;AAAA,EACpB,UAAU,CAAC;AAAA,EACX,aAAa,CAAC;AAAA,EACd,UAAU,CAAC;AAAA,EACX,qBAAqB,CAAC;AAAA,EACtB,cAAc,CAAC;AAChB;AASA,SAAS,OAAO;AACf,SAAO,KAAK,MAAM,KAAK,MAAM,KAAK,IAAI,IAAI,GAAI,IAAI,EAAE;AACrD;AAEO,MAAM,QAAQ;AAAA,EAIpB,YAAY,KAAa;AAFzB,oBAAW,KAAK;AAChB,iBAAQ;AAEP,SAAK,MAAM;AAAA,EACZ;AAAA,EACA,gBAAgB;AACf,UAAM,MAAM,KAAK;AACjB,QAAI,KAAK,aAAa,KAAK;AAC1B,WAAK,QAAQ;AACb,WAAK,WAAW;AAAA,IACjB;AACA,SAAK;AACL,WAAO,KAAK,QAAQ,KAAK;AAAA,EAC1B;AACD;AAEA,SAAS,SAAS,SAAiB;AAClC,YAAU,QAAQ,YAAY,EAAE,QAAQ,cAAc,EAAE;AACxD,SAAO,CAAC,MAAM,MAAM,QAAQ,MAAM,MAAM,QAAQ,OAAO,EAAE,SAAS,OAAO;AAC1E;AAEA,IAAI,eAA8B;AAC3B,MAAM,UAAU,IAAI,QAAQ,GAAG;AAC/B,MAAM,KAAK,IAAI,0BAAe,oBAA2D,QAAQ,OAAM,SAAQ;AACrH,MAAI,SAAS,IAAI,KAAK,CAAC,QAAQ,cAAc;AAAG,WAAO;AACvD,MAAI,gBAAiB,KAAK,IAAI,IAAI,eAAe,KAAQ;AACxD,WAAO;AAAA,EACR;AACA,MAAI;AAAc,mBAAe;AAEjC,QAAM,cAAkC;AAAA;AAAA,IAEvC,WAAW,CAAC,IAAI;AAAA,IAChB,qBAAqB;AAAA,IACrB,SAAS,EAAC,KAAI;AAAA,EACf;AACA,MAAI;AACH,UAAM,MAAM,UAAM,gBAAI,kEAAkE,EAAE,KAAK;AAAA,MAC9F,OAAO;AAAA,QACN,KAAK,4BAAO;AAAA,MACb;AAAA,MACA,MAAM,KAAK,UAAU,WAAW;AAAA,MAChC,SAAS;AAAA,QACR,gBAAgB;AAAA,MACjB;AAAA,MACA,SAAS,KAAK;AAAA;AAAA,IACf,CAAC;AACD,QAAI,CAAC;AAAK,aAAO;AACjB,UAAM,OAAO,KAAK,MAAM,GAAG;AAC3B,QAAI,KAAK;AAAO,YAAM,IAAI,MAAM,KAAK,OAAO;AAC5C,UAAM,SAAgC,CAAC;AACvC,eAAW,KAAK,KAAK,iBAAiB;AACrC,YAAM,QAAQ,KAAK,gBAAgB,CAAC;AACpC,aAAO,CAAC,IAAI,MAAM,aAAa;AAAA,IAChC;AACA,WAAO;AAAA,EACR,SAAS,GAAP;AACD,mBAAe,KAAK,IAAI;AACxB,QAAI,EAAE,QAAQ,WAAW,iBAAiB,GAAG;AAG5C,aAAO;AAAA,IACR;AACA,YAAQ,SAAS,GAAG,6BAA6B,EAAC,SAAS,KAAK,UAAU,WAAW,EAAC,CAAC;AACvF,WAAO;AAAA,EACR;AACD,GAAG,UAAU;AAKb,IAAI,QAAQ,SAAS,QAAQ;AAE5B,SAAO,SAAS;AAChB,SAAO,UAAU;AAAA,IAChB,SAAS,OAAc,SAAS,kCAAkC,UAA4B,MAAM;AACnG,YAAM,OAAO,KAAK,UAAU,CAAC,MAAM,MAAM,MAAM,SAAS,QAAQ,OAAO,CAAC;AACxE,cAAQ,KAAM;AAAA,MAAc;AAAA,EAAS,MAAM,OAAO;AAAA,IACnD;AAAA,IACA,KAAK,MAAc;AAClB,cAAQ,KAAM;AAAA;AAAA,EAAmB,MAAM;AAAA,IACxC;AAAA,EACD;AACA,SAAO,OAAO;AACd,UAAQ,GAAG,qBAAqB,SAAO;AACtC,QAAI,4BAAO,YAAY;AACtB,cAAQ,SAAS,KAAK,gCAAgC;AAAA,IACvD;AAAA,EACD,CAAC;AAED,kBAAK,MAAM,uBAAuB,QAAQ,OAAO,SAAO,KAAK,GAAG,CAAC;AAClE,WAAW,CAAC,QAAQ,MAAM;AACzB,KAAG,MAAM,4BAAO,0BAA0B,CAAC;AAC5C;AAEO,MAAM,iBAAiB;AAAA,EAG7B,SAAS,MAAc;AACtB,QAAI,CAAC,4BAAO;AAAgB,aAAO,QAAQ,QAAQ,IAAI;AACvD,WAAO,GAAG,MAAM,IAAI;AAAA,EACrB;AAAA,EACA,MAAM,aAAa,MAAc,MAA8B;AAC9D,QAAI,CAAC,4BAAO;AAAgB,aAAO,QAAQ,QAAQ,IAAI;AACvD,UAAM,OAAkB;AAAA,MACvB,SAAS,EAAC,KAAI;AAAA,MACd,iBAAiB,CAAC;AAAA,IACnB;AACA,eAAW,KAAK,MAAM;AACrB,WAAK,gBAAgB,CAAC,IAAI,EAAC,cAAc,EAAC,OAAO,KAAK,CAAC,EAAC,EAAC;AAAA,IAC1D;AACA,QAAI;AACH,YAAM,MAAM,UAAM,gBAAI,uEAAuE,EAAE,KAAK;AAAA,QACnG,OAAO;AAAA,UACN,KAAK,4BAAO;AAAA,QACb;AAAA,QACA,MAAM,KAAK,UAAU,IAAI;AAAA,QACzB,SAAS;AAAA,UACR,gBAAgB;AAAA,QACjB;AAAA,QACA,SAAS,KAAK;AAAA;AAAA,MACf,CAAC;AACD,aAAO,KAAK,MAAM,GAAG;AAAA,IACtB,SAAS,GAAP;AACD,aAAO,EAAC,OAAO,EAAE,QAAO;AAAA,IACzB;AAAA,EACD;AAAA,EACA,UAAU;AACT,WAAO,GAAG,QAAQ;AAAA,EACnB;AAAA,EACA,UAAU;AACT,WAAO,GAAG,QAAQ;AAAA,EACnB;AAAA,EACA,MAAM,QAAgB;AACrB,OAAG,MAAM,MAAM;AAAA,EAChB;AAAA,EACA,qBAAqB;AACpB,WAAO,GAAG,UAAU;AAAA,EACrB;AACD;AA5Ca,iBACI,KAAK;AADT,iBAEI,aAAa;",
"names": []
}