From feab5f8cd05b7b18d027662df80e21b27ac08068 Mon Sep 17 00:00:00 2001 From: Siddharth Singh Date: Sun, 6 Apr 2025 15:20:06 +0530 Subject: [PATCH 01/10] stop logging every single query --- packages/db/KyselyLogger.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/db/KyselyLogger.ts b/packages/db/KyselyLogger.ts index 333e4285..6d0752e6 100644 --- a/packages/db/KyselyLogger.ts +++ b/packages/db/KyselyLogger.ts @@ -14,7 +14,7 @@ export const KyselyLogger: Logger = (event) => { dbQueryDurationHistogram.observe(duration); if (event.level === 'query') { - logi({ level: 'debug', ns: 'ditto.sql', sql, parameters: parameters as LogiValue, duration }); + logi({ level: 'trace', ns: 'ditto.sql', sql, parameters: parameters as LogiValue, duration }); } if (event.level === 'error') { From 5dc2cc405fe4cb55a9d8ac0ef83adbbf2712393a Mon Sep 17 00:00:00 2001 From: Siddharth Singh Date: Sun, 6 Apr 2025 15:20:17 +0530 Subject: [PATCH 02/10] add log configuration variables --- packages/conf/DittoConf.ts | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/packages/conf/DittoConf.ts b/packages/conf/DittoConf.ts index 59a3fde4..8373a5b6 100644 --- a/packages/conf/DittoConf.ts +++ b/packages/conf/DittoConf.ts @@ -238,6 +238,19 @@ export class DittoConf { }; } + get isCI() { + return this.env.get('CI') === 'true'; + } + + get logConfig() { + const [fmt = 'jsonl', level = 'debug', scopes = ''] = (this.env.get('DEBUG') || '').split(':'); + return { + fmt, + level, + scopes: scopes.split(',').filter(Boolean), + }; + } + /** nostr.build API endpoint when the `nostrbuild` uploader is used. */ get nostrbuildEndpoint(): string { return this.env.get('NOSTRBUILD_ENDPOINT') || 'https://nostr.build/api/v2/upload/files'; From 6778a42d54d3cbbdadf67816ed1a5c07ccf02f3f Mon Sep 17 00:00:00 2001 From: Siddharth Singh Date: Sun, 6 Apr 2025 15:20:39 +0530 Subject: [PATCH 03/10] remove isCI conf var --- packages/conf/DittoConf.ts | 4 ---- 1 file changed, 4 deletions(-) diff --git a/packages/conf/DittoConf.ts b/packages/conf/DittoConf.ts index 8373a5b6..8a79684d 100644 --- a/packages/conf/DittoConf.ts +++ b/packages/conf/DittoConf.ts @@ -238,10 +238,6 @@ export class DittoConf { }; } - get isCI() { - return this.env.get('CI') === 'true'; - } - get logConfig() { const [fmt = 'jsonl', level = 'debug', scopes = ''] = (this.env.get('DEBUG') || '').split(':'); return { From 82e7f854b94e236468fc2f9f75aee94caf03daad Mon Sep 17 00:00:00 2001 From: Siddharth Singh Date: Sun, 6 Apr 2025 15:22:45 +0530 Subject: [PATCH 04/10] replace logi handler --- packages/ditto/app.ts | 3 ++ packages/ditto/utils/logi.ts | 71 ++++++++++++++++++++++++++++++++++++ 2 files changed, 74 insertions(+) create mode 100644 packages/ditto/utils/logi.ts diff --git a/packages/ditto/app.ts b/packages/ditto/app.ts index b4ecbaec..857bae60 100644 --- a/packages/ditto/app.ts +++ b/packages/ditto/app.ts @@ -152,6 +152,8 @@ import dittoNamesRoute from '@/routes/dittoNamesRoute.ts'; import pleromaAdminPermissionGroupsRoute from '@/routes/pleromaAdminPermissionGroupsRoute.ts'; import pleromaStatusesRoute from '@/routes/pleromaStatusesRoute.ts'; import { DittoRelayStore } from '@/storages/DittoRelayStore.ts'; +import { logi } from '@soapbox/logi'; +import { createLogiHandler } from '@/utils/logi.ts'; export interface AppEnv extends DittoEnv { Variables: DittoEnv['Variables'] & { @@ -179,6 +181,7 @@ type AppMiddleware = MiddlewareHandler; type AppController

= Handler>; const conf = new DittoConf(Deno.env); +logi.handler = createLogiHandler(conf, logi.handler); startSentry(conf); diff --git a/packages/ditto/utils/logi.ts b/packages/ditto/utils/logi.ts new file mode 100644 index 00000000..18305459 --- /dev/null +++ b/packages/ditto/utils/logi.ts @@ -0,0 +1,71 @@ +import type { LogiHandler, LogiLog, LogiValue } from '@soapbox/logi'; +import { DittoConf } from '@ditto/conf'; + +type Level = LogiLog['level']; + +const levels: Level[] = ['trace', 'debug', 'info', 'warn', 'error', 'fatal', 'critical']; + +const lowerLevels: Record = levels.reduce((acc, level, index) => { + acc[level] = levels.slice(index); + return acc; +}, {} as Record); + +const colors: Record = { + trace: 'grey', + debug: 'white', + info: 'blue', + warn: 'yellow', + error: 'orange', + fatal: 'red', + critical: 'red', +}; + +const levelSet = new Set(levels); +const isLevel = (str: string): str is Level => levelSet.has(str as Level); + +const prettyPrint = (msg: LogiValue): string => { + const message = msg || ''; + const type = typeof message; + switch (type) { + case 'string': + case 'bigint': + case 'number': + case 'boolean': + return message.toString(); + case 'function': + case 'symbol': + case 'undefined': + return `<${type}>`; + case 'object': + if (message === null) return ""; + return JSON.stringify(message, (_, v) => { + if (Array.isArray(v)) { + return `[${v.map(itm => JSON.stringify(itm)).join(', ')}]`; + } + if (typeof v === 'string') return `\`${v}\``; + return v; + }, 2) + .replaceAll('\\"', '"') + .replace(/^"/, '') + .replace(/"$/, ''); + } +} + +const pair = (key: string, value: LogiValue | undefined) => { + return `${key} => ${prettyPrint(value || '')}` +} + +export const createLogiHandler = (conf: DittoConf, defaultHandler: LogiHandler) => (log: LogiLog) => { + const { fmt, level, scopes } = conf.logConfig; + if (fmt === 'jsonl') return defaultHandler(log); + if (!isLevel(level)) throw new Error(`Invalid log level ${level} specified`); + if (!lowerLevels[level].includes(log.level)) return; + if (scopes.length && !scopes.includes(log.ns)) return; + const message = prettyPrint(log.message || log.msg || ''); + const remaining = Object.entries(log) + .filter(([key]) => !['ns', 'level', 'message', 'msg'].includes(key)); + + console.group(`%c${log.level.toUpperCase()} %c${log.ns} %c${message || ''}`, `color: ${colors[log.level]}; font-weight: bold`, 'font-weight: normal; color: yellow', 'color: unset'); + console.log(remaining.map(itm => pair(...itm)).join('\n')); + console.groupEnd(); +}; From d3a15a699ecccb873b9a902c9ac39e91b6b59438 Mon Sep 17 00:00:00 2001 From: Siddharth Singh Date: Sun, 6 Apr 2025 15:33:36 +0530 Subject: [PATCH 05/10] fmt --- packages/ditto/utils/logi.ts | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/packages/ditto/utils/logi.ts b/packages/ditto/utils/logi.ts index 18305459..f977da3b 100644 --- a/packages/ditto/utils/logi.ts +++ b/packages/ditto/utils/logi.ts @@ -37,10 +37,10 @@ const prettyPrint = (msg: LogiValue): string => { case 'undefined': return `<${type}>`; case 'object': - if (message === null) return ""; + if (message === null) return ''; return JSON.stringify(message, (_, v) => { if (Array.isArray(v)) { - return `[${v.map(itm => JSON.stringify(itm)).join(', ')}]`; + return `[${v.map((itm) => JSON.stringify(itm)).join(', ')}]`; } if (typeof v === 'string') return `\`${v}\``; return v; @@ -49,11 +49,11 @@ const prettyPrint = (msg: LogiValue): string => { .replace(/^"/, '') .replace(/"$/, ''); } -} +}; const pair = (key: string, value: LogiValue | undefined) => { - return `${key} => ${prettyPrint(value || '')}` -} + return `${key} => ${prettyPrint(value || '')}`; +}; export const createLogiHandler = (conf: DittoConf, defaultHandler: LogiHandler) => (log: LogiLog) => { const { fmt, level, scopes } = conf.logConfig; @@ -65,7 +65,13 @@ export const createLogiHandler = (conf: DittoConf, defaultHandler: LogiHandler) const remaining = Object.entries(log) .filter(([key]) => !['ns', 'level', 'message', 'msg'].includes(key)); - console.group(`%c${log.level.toUpperCase()} %c${log.ns} %c${message || ''}`, `color: ${colors[log.level]}; font-weight: bold`, 'font-weight: normal; color: yellow', 'color: unset'); - console.log(remaining.map(itm => pair(...itm)).join('\n')); + console.group( + `%c${log.level.toUpperCase()} %c${log.ns} %c${message || ''}`, + `color: ${colors[log.level]}; font-weight: bold`, + 'font-weight: normal; color: yellow', + 'color: unset', + ); + + console.log(remaining.map((itm) => pair(...itm)).join('\n')); console.groupEnd(); }; From 1b19ff415a390f8fe7fd853a1a644c05204c3adc Mon Sep 17 00:00:00 2001 From: Siddharth Singh Date: Sun, 6 Apr 2025 15:40:42 +0530 Subject: [PATCH 06/10] add types to public api for dittoconf --- packages/conf/DittoConf.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/packages/conf/DittoConf.ts b/packages/conf/DittoConf.ts index 8a79684d..6a199ae9 100644 --- a/packages/conf/DittoConf.ts +++ b/packages/conf/DittoConf.ts @@ -238,10 +238,14 @@ export class DittoConf { }; } - get logConfig() { + get logConfig(): { + fmt: 'jsonl' | 'pretty'; + level: string; + scopes: string[]; + } { const [fmt = 'jsonl', level = 'debug', scopes = ''] = (this.env.get('DEBUG') || '').split(':'); return { - fmt, + fmt: fmt === 'jsonl' ? fmt : 'pretty', level, scopes: scopes.split(',').filter(Boolean), }; From 8f51a9d6d7937a02134c0b03bee1fe93ebfd67bb Mon Sep 17 00:00:00 2001 From: Siddharth Singh Date: Sun, 6 Apr 2025 15:46:24 +0530 Subject: [PATCH 07/10] only show extra params when they are actually present --- packages/conf/DittoConf.ts | 7 +++++++ packages/ditto/utils/logi.ts | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/packages/conf/DittoConf.ts b/packages/conf/DittoConf.ts index 6a199ae9..6c06714c 100644 --- a/packages/conf/DittoConf.ts +++ b/packages/conf/DittoConf.ts @@ -238,6 +238,13 @@ export class DittoConf { }; } + /** + * The logging configuration for the Ditto server. The config is derived from + * the DEBUG environment variable and it is parsed as follows: + * + * `DEBUG='::comma-separated scopes to show'`. + * If the scopes are empty (e.g. in 'pretty:warn:', then all scopes are shown.) + */ get logConfig(): { fmt: 'jsonl' | 'pretty'; level: string; diff --git a/packages/ditto/utils/logi.ts b/packages/ditto/utils/logi.ts index f977da3b..f2db4256 100644 --- a/packages/ditto/utils/logi.ts +++ b/packages/ditto/utils/logi.ts @@ -72,6 +72,6 @@ export const createLogiHandler = (conf: DittoConf, defaultHandler: LogiHandler) 'color: unset', ); - console.log(remaining.map((itm) => pair(...itm)).join('\n')); + if (remaining.length) console.log(remaining.map((itm) => pair(...itm)).join('\n')); console.groupEnd(); }; From a1aed97decd7ddfaa861c69e3f4cccc4a69bf983 Mon Sep 17 00:00:00 2001 From: Siddharth Singh Date: Sun, 6 Apr 2025 15:50:25 +0530 Subject: [PATCH 08/10] enable all subscopes of a given scope when it's supplied --- packages/ditto/utils/logi.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/ditto/utils/logi.ts b/packages/ditto/utils/logi.ts index f2db4256..749260c7 100644 --- a/packages/ditto/utils/logi.ts +++ b/packages/ditto/utils/logi.ts @@ -60,7 +60,7 @@ export const createLogiHandler = (conf: DittoConf, defaultHandler: LogiHandler) if (fmt === 'jsonl') return defaultHandler(log); if (!isLevel(level)) throw new Error(`Invalid log level ${level} specified`); if (!lowerLevels[level].includes(log.level)) return; - if (scopes.length && !scopes.includes(log.ns)) return; + if (scopes.length && !scopes.some(scope => scope.startsWith(log.ns))) return; const message = prettyPrint(log.message || log.msg || ''); const remaining = Object.entries(log) .filter(([key]) => !['ns', 'level', 'message', 'msg'].includes(key)); From 8305cea81e7e9f81010ad0003feaccc9a3a39c7c Mon Sep 17 00:00:00 2001 From: Siddharth Singh Date: Sun, 6 Apr 2025 15:53:06 +0530 Subject: [PATCH 09/10] fmt ... again --- packages/ditto/utils/logi.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/ditto/utils/logi.ts b/packages/ditto/utils/logi.ts index 749260c7..d8ba9588 100644 --- a/packages/ditto/utils/logi.ts +++ b/packages/ditto/utils/logi.ts @@ -60,7 +60,7 @@ export const createLogiHandler = (conf: DittoConf, defaultHandler: LogiHandler) if (fmt === 'jsonl') return defaultHandler(log); if (!isLevel(level)) throw new Error(`Invalid log level ${level} specified`); if (!lowerLevels[level].includes(log.level)) return; - if (scopes.length && !scopes.some(scope => scope.startsWith(log.ns))) return; + if (scopes.length && !scopes.some((scope) => scope.startsWith(log.ns))) return; const message = prettyPrint(log.message || log.msg || ''); const remaining = Object.entries(log) .filter(([key]) => !['ns', 'level', 'message', 'msg'].includes(key)); From 1b559321937022c560616ed1a07c8c226c60154f Mon Sep 17 00:00:00 2001 From: Siddharth Singh Date: Tue, 8 Apr 2025 17:09:05 +0530 Subject: [PATCH 10/10] apply review fixes (thanks @alexgleason) --- packages/conf/DittoConf.ts | 2 +- packages/ditto/utils/logi.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/conf/DittoConf.ts b/packages/conf/DittoConf.ts index 6c06714c..09e6eed2 100644 --- a/packages/conf/DittoConf.ts +++ b/packages/conf/DittoConf.ts @@ -250,7 +250,7 @@ export class DittoConf { level: string; scopes: string[]; } { - const [fmt = 'jsonl', level = 'debug', scopes = ''] = (this.env.get('DEBUG') || '').split(':'); + const [fmt = 'jsonl', level = 'debug', scopes = ''] = (this.env.get('LOG_CONFIG') || '').split(':'); return { fmt: fmt === 'jsonl' ? fmt : 'pretty', level, diff --git a/packages/ditto/utils/logi.ts b/packages/ditto/utils/logi.ts index d8ba9588..4b764201 100644 --- a/packages/ditto/utils/logi.ts +++ b/packages/ditto/utils/logi.ts @@ -52,7 +52,7 @@ const prettyPrint = (msg: LogiValue): string => { }; const pair = (key: string, value: LogiValue | undefined) => { - return `${key} => ${prettyPrint(value || '')}`; + return `${key}: ${prettyPrint(value || '')}`; }; export const createLogiHandler = (conf: DittoConf, defaultHandler: LogiHandler) => (log: LogiLog) => {