From 6ff90d63bbdaa139e0cfba99971653169909ad9b Mon Sep 17 00:00:00 2001 From: Siddharth Singh Date: Sun, 25 Aug 2024 16:23:25 +0530 Subject: [PATCH 01/82] implement PUT /api/v1/media/:id Adds support for setting image descriptions for accessibility reasons --- src/app.ts | 6 +++++- src/controllers/api/media.ts | 28 +++++++++++++++++++++++++++- src/db/unattached-media.ts | 13 +++++++++++++ src/views/mastodon/attachments.ts | 2 +- 4 files changed, 46 insertions(+), 3 deletions(-) diff --git a/src/app.ts b/src/app.ts index 9828724d..a0e752ba 100644 --- a/src/app.ts +++ b/src/app.ts @@ -53,7 +53,7 @@ import { instanceV2Controller, } from '@/controllers/api/instance.ts'; import { markersController, updateMarkersController } from '@/controllers/api/markers.ts'; -import { mediaController } from '@/controllers/api/media.ts'; +import { mediaController, updateMediaDescriptionController } from '@/controllers/api/media.ts'; import { mutesController } from '@/controllers/api/mutes.ts'; import { notificationsController } from '@/controllers/api/notifications.ts'; import { createTokenController, oauthAuthorizeController, oauthController } from '@/controllers/api/oauth.ts'; @@ -226,6 +226,10 @@ app.delete('/api/v1/statuses/:id{[0-9a-f]{64}}', requireSigner, deleteStatusCont app.get('/api/v1/pleroma/statuses/:id{[0-9a-f]{64}}/quotes', quotesController); app.post('/api/v1/media', mediaController); +app.put( + '/api/v1/media/:id{[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}}', + updateMediaDescriptionController, +); app.post('/api/v2/media', mediaController); app.get('/api/v1/timelines/home', requireSigner, homeTimelineController); diff --git a/src/controllers/api/media.ts b/src/controllers/api/media.ts index 71b3e782..f0ed3069 100644 --- a/src/controllers/api/media.ts +++ b/src/controllers/api/media.ts @@ -5,6 +5,7 @@ import { fileSchema } from '@/schema.ts'; import { parseBody } from '@/utils/api.ts'; import { renderAttachment } from '@/views/mastodon/attachments.ts'; import { uploadFile } from '@/utils/upload.ts'; +import { setMediaDescription } from '@/db/unattached-media.ts'; const mediaBodySchema = z.object({ file: fileSchema, @@ -13,6 +14,10 @@ const mediaBodySchema = z.object({ focus: z.string().optional(), }); +const mediaDescriptionUpdateSchema = z.object({ + description: z.string(), +}); + const mediaController: AppController = async (c) => { const pubkey = await c.get('signer')?.getPublicKey()!; const result = mediaBodySchema.safeParse(await parseBody(c.req.raw)); @@ -32,4 +37,25 @@ const mediaController: AppController = async (c) => { } }; -export { mediaController }; +const updateMediaDescriptionController: AppController = async (c) => { + console.log('in media description update controller'); + const result = mediaDescriptionUpdateSchema.safeParse(await parseBody(c.req.raw)); + console.log(result); + if (!result.success) { + return c.json({ error: 'Bad request.', schema: result.error }, 422); + } + try { + const { description } = result.data; + console.log(description); + if (!await setMediaDescription(c.req.param('id'), description)) { + return c.json({ error: 'File with specified ID not found.' }, 404); + } + } catch (e) { + console.error(e); + return c.json({ error: 'Failed to set media description.' }, 500); + } + + return c.json({ message: 'ok' }, 200); +}; + +export { mediaController, updateMediaDescriptionController }; diff --git a/src/db/unattached-media.ts b/src/db/unattached-media.ts index fac7a1d9..260a9d5a 100644 --- a/src/db/unattached-media.ts +++ b/src/db/unattached-media.ts @@ -56,6 +56,18 @@ async function getUnattachedMediaByIds(kysely: Kysely, ids: string[ })); } +async function setMediaDescription(id: string, desc = '') { + const { kysely } = await DittoDB.getInstance(); + const existing = await selectUnattachedMediaQuery(kysely).where('id', '=', id).executeTakeFirst(); + if (!existing) return false; + const parsed = (await JSON.parse(existing.data) as string[][]).filter((itm) => itm[0] !== 'alt'); + parsed.push(['alt', desc]); + await kysely.updateTable('unattached_media') + .set({ data: JSON.stringify(parsed) }) + .execute(); + return true; +} + /** Delete rows as an event with media is being created. */ async function deleteAttachedMedia(pubkey: string, urls: string[]): Promise { if (!urls.length) return; @@ -71,5 +83,6 @@ export { deleteUnattachedMediaByUrl, getUnattachedMediaByIds, insertUnattachedMedia, + setMediaDescription, type UnattachedMedia, }; diff --git a/src/views/mastodon/attachments.ts b/src/views/mastodon/attachments.ts index 9320f604..293b1499 100644 --- a/src/views/mastodon/attachments.ts +++ b/src/views/mastodon/attachments.ts @@ -10,7 +10,7 @@ function renderAttachment( const url = tags.find(([name]) => name === 'url')?.[1]; const m = tags.find(([name]) => name === 'm')?.[1] ?? getUrlMediaType(url!); - const alt = tags.find(([name]) => name === 'alt')?.[1]; + const alt = tags.find(([name]) => name === 'alt')?.slice(1).join(' ') || 'picture'; const cid = tags.find(([name]) => name === 'cid')?.[1]; const dim = tags.find(([name]) => name === 'dim')?.[1]; const blurhash = tags.find(([name]) => name === 'blurhash')?.[1]; From cf4dc8627b8fb50c724f680bd47174fd76fe8065 Mon Sep 17 00:00:00 2001 From: Siddharth Singh Date: Sun, 25 Aug 2024 16:25:36 +0530 Subject: [PATCH 02/82] remove unnecessary coalescing --- src/views/mastodon/attachments.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/views/mastodon/attachments.ts b/src/views/mastodon/attachments.ts index 293b1499..9307ea82 100644 --- a/src/views/mastodon/attachments.ts +++ b/src/views/mastodon/attachments.ts @@ -10,7 +10,7 @@ function renderAttachment( const url = tags.find(([name]) => name === 'url')?.[1]; const m = tags.find(([name]) => name === 'm')?.[1] ?? getUrlMediaType(url!); - const alt = tags.find(([name]) => name === 'alt')?.slice(1).join(' ') || 'picture'; + const alt = tags.find(([name]) => name === 'alt')?.slice(1).join(' '); const cid = tags.find(([name]) => name === 'cid')?.[1]; const dim = tags.find(([name]) => name === 'dim')?.[1]; const blurhash = tags.find(([name]) => name === 'blurhash')?.[1]; From 1f04a7fcdfb914c8320344a6653375177a58763f Mon Sep 17 00:00:00 2001 From: Siddharth Singh Date: Sun, 25 Aug 2024 17:20:25 +0530 Subject: [PATCH 03/82] remove debug console.logs --- src/controllers/api/media.ts | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/controllers/api/media.ts b/src/controllers/api/media.ts index f0ed3069..5b0cd218 100644 --- a/src/controllers/api/media.ts +++ b/src/controllers/api/media.ts @@ -38,15 +38,12 @@ const mediaController: AppController = async (c) => { }; const updateMediaDescriptionController: AppController = async (c) => { - console.log('in media description update controller'); const result = mediaDescriptionUpdateSchema.safeParse(await parseBody(c.req.raw)); - console.log(result); if (!result.success) { return c.json({ error: 'Bad request.', schema: result.error }, 422); } try { const { description } = result.data; - console.log(description); if (!await setMediaDescription(c.req.param('id'), description)) { return c.json({ error: 'File with specified ID not found.' }, 404); } From f8fae52d5e083a4676989ef1604f57d38c88adb3 Mon Sep 17 00:00:00 2001 From: Siddharth Singh Date: Sun, 25 Aug 2024 19:13:31 +0530 Subject: [PATCH 04/82] fix bug in alt text tagging --- src/views/mastodon/attachments.ts | 2 +- src/views/mastodon/statuses.ts | 7 ++++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/src/views/mastodon/attachments.ts b/src/views/mastodon/attachments.ts index 9307ea82..9320f604 100644 --- a/src/views/mastodon/attachments.ts +++ b/src/views/mastodon/attachments.ts @@ -10,7 +10,7 @@ function renderAttachment( const url = tags.find(([name]) => name === 'url')?.[1]; const m = tags.find(([name]) => name === 'm')?.[1] ?? getUrlMediaType(url!); - const alt = tags.find(([name]) => name === 'alt')?.slice(1).join(' '); + const alt = tags.find(([name]) => name === 'alt')?.[1]; const cid = tags.find(([name]) => name === 'cid')?.[1]; const dim = tags.find(([name]) => name === 'dim')?.[1]; const blurhash = tags.find(([name]) => name === 'blurhash')?.[1]; diff --git a/src/views/mastodon/statuses.ts b/src/views/mastodon/statuses.ts index d1c02f1e..bc42765b 100644 --- a/src/views/mastodon/statuses.ts +++ b/src/views/mastodon/statuses.ts @@ -84,7 +84,12 @@ async function renderStatus(event: DittoEvent, opts: RenderStatusOpts): Promise< const imeta: string[][][] = event.tags .filter(([name]) => name === 'imeta') - .map(([_, ...entries]) => entries.map((entry) => entry.split(' '))); + .map(([_, ...entries]) => + entries.map((entry) => { + const split = entry.split(' '); + return [split[0], split.splice(1).join(' ')]; + }) + ); const media = imeta.length ? imeta : getMediaLinks(links); From 7e711aa8a8858995247f4e01dcd65ba49db0aa5d Mon Sep 17 00:00:00 2001 From: Siddharth Singh Date: Sun, 25 Aug 2024 19:43:28 +0530 Subject: [PATCH 05/82] updateMediaDescriptionController --> updateMediaController --- src/app.ts | 4 ++-- src/controllers/api/media.ts | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/app.ts b/src/app.ts index a0e752ba..43ad4b00 100644 --- a/src/app.ts +++ b/src/app.ts @@ -53,7 +53,7 @@ import { instanceV2Controller, } from '@/controllers/api/instance.ts'; import { markersController, updateMarkersController } from '@/controllers/api/markers.ts'; -import { mediaController, updateMediaDescriptionController } from '@/controllers/api/media.ts'; +import { mediaController, updateMediaController } from '@/controllers/api/media.ts'; import { mutesController } from '@/controllers/api/mutes.ts'; import { notificationsController } from '@/controllers/api/notifications.ts'; import { createTokenController, oauthAuthorizeController, oauthController } from '@/controllers/api/oauth.ts'; @@ -228,7 +228,7 @@ app.get('/api/v1/pleroma/statuses/:id{[0-9a-f]{64}}/quotes', quotesController); app.post('/api/v1/media', mediaController); app.put( '/api/v1/media/:id{[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}}', - updateMediaDescriptionController, + updateMediaController, ); app.post('/api/v2/media', mediaController); diff --git a/src/controllers/api/media.ts b/src/controllers/api/media.ts index 5b0cd218..e13e9d21 100644 --- a/src/controllers/api/media.ts +++ b/src/controllers/api/media.ts @@ -37,7 +37,7 @@ const mediaController: AppController = async (c) => { } }; -const updateMediaDescriptionController: AppController = async (c) => { +const updateMediaController: AppController = async (c) => { const result = mediaDescriptionUpdateSchema.safeParse(await parseBody(c.req.raw)); if (!result.success) { return c.json({ error: 'Bad request.', schema: result.error }, 422); @@ -55,4 +55,4 @@ const updateMediaDescriptionController: AppController = async (c) => { return c.json({ message: 'ok' }, 200); }; -export { mediaController, updateMediaDescriptionController }; +export { mediaController, updateMediaController }; From 1a38061b34a070bcd6b8edaa37ffe2d2eea04f69 Mon Sep 17 00:00:00 2001 From: Siddharth Singh Date: Sun, 25 Aug 2024 19:48:04 +0530 Subject: [PATCH 06/82] mediaDescriptionUpdateSchema --> mediaUpdateSchema --- src/controllers/api/media.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/controllers/api/media.ts b/src/controllers/api/media.ts index e13e9d21..3fa3ee68 100644 --- a/src/controllers/api/media.ts +++ b/src/controllers/api/media.ts @@ -14,7 +14,7 @@ const mediaBodySchema = z.object({ focus: z.string().optional(), }); -const mediaDescriptionUpdateSchema = z.object({ +const mediaUpdateSchema = z.object({ description: z.string(), }); @@ -38,7 +38,7 @@ const mediaController: AppController = async (c) => { }; const updateMediaController: AppController = async (c) => { - const result = mediaDescriptionUpdateSchema.safeParse(await parseBody(c.req.raw)); + const result = mediaUpdateSchema.safeParse(await parseBody(c.req.raw)); if (!result.success) { return c.json({ error: 'Bad request.', schema: result.error }, 422); } From 8e53da7980304f0bcf219ab554e8e2ea176b6436 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Thu, 29 Aug 2024 10:22:24 -0300 Subject: [PATCH 07/82] feat: add clean type errors task --- deno.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/deno.json b/deno.json index 1b035b20..b90373bf 100644 --- a/deno.json +++ b/deno.json @@ -18,7 +18,8 @@ "setup": "deno run -A scripts/setup.ts", "stats:recompute": "deno run -A scripts/stats-recompute.ts", "soapbox": "curl -O https://dl.soapbox.pub/main/soapbox.zip && mkdir -p public && mv soapbox.zip public/ && cd public/ && unzip soapbox.zip && rm soapbox.zip", - "trends": "deno run -A scripts/trends.ts" + "trends": "deno run -A scripts/trends.ts", + "clean:type-errors": "deno cache --reload src/app.ts" }, "unstable": ["cron", "ffi", "kv", "worker-options"], "exclude": ["./public"], From e7f5535bd2eda62d41d80f1ce57bbd3b11907d42 Mon Sep 17 00:00:00 2001 From: Siddharth Singh Date: Sun, 1 Sep 2024 19:15:22 +0530 Subject: [PATCH 08/82] handle event insertion errors with warning --- scripts/nostr-pull.ts | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/scripts/nostr-pull.ts b/scripts/nostr-pull.ts index 68766c8f..0556b64a 100644 --- a/scripts/nostr-pull.ts +++ b/scripts/nostr-pull.ts @@ -29,6 +29,18 @@ const importUsers = async ( const notes = new Set(); const { profilesOnly = false } = opts || {}; + const put = async (event: NostrEvent) => { + try { + await doEvent(event); + } catch (error) { + if (error.message.includes('violates unique constraint')) { + console.warn(`Skipping existing event ${event.id}...`); + } else { + console.error(error); + } + } + }; + await Promise.all(relays.map(async (relay) => { if (!relay.startsWith('wss://')) console.error(`Invalid relay url ${relay}`); const conn = new NRelay1(relay); @@ -49,7 +61,7 @@ const importUsers = async ( if (kind === 1 && !notes.has(event.id)) { // add the event to eventsDB only if it has not been found already. notes.add(event.id); - await doEvent(event); + await put(event); return; } @@ -64,7 +76,7 @@ const importUsers = async ( for (const user in profiles) { const profile = profiles[user]; for (const kind in profile) { - await doEvent(profile[kind]); + await put(profile[kind]); } let name = user; From 729471d6927244c5b6e67d4970ceb5762a255ce1 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Mon, 2 Sep 2024 09:52:34 -0300 Subject: [PATCH 09/82] feat(notifications api): implement zap notification calls database for zap events --- src/controllers/api/notifications.ts | 51 ++++++++++++++++++++++++++-- 1 file changed, 48 insertions(+), 3 deletions(-) diff --git a/src/controllers/api/notifications.ts b/src/controllers/api/notifications.ts index 54e88edf..00ac9930 100644 --- a/src/controllers/api/notifications.ts +++ b/src/controllers/api/notifications.ts @@ -4,9 +4,10 @@ import { z } from 'zod'; import { AppContext, AppController } from '@/app.ts'; import { Conf } from '@/config.ts'; import { DittoPagination } from '@/interfaces/DittoPagination.ts'; +import { getAmount } from '@/utils/bolt11.ts'; import { hydrateEvents } from '@/storages/hydrate.ts'; import { paginated } from '@/utils/api.ts'; -import { renderNotification } from '@/views/mastodon/notifications.ts'; +import { renderNotification, RenderNotificationOpts } from '@/views/mastodon/notifications.ts'; /** Set of known notification types across backends. */ const notificationTypes = new Set([ @@ -23,6 +24,7 @@ const notificationTypes = new Set([ 'severed_relationships', 'pleroma:emoji_reaction', 'ditto:name_grant', + 'ditto:zap', ]); const notificationsSchema = z.object({ @@ -67,6 +69,10 @@ const notificationsController: AppController = async (c) => { filters.push({ kinds: [30360], authors: [Conf.pubkey], '#p': [pubkey], ...params }); } + if (types.has('ditto:zap')) { + filters.push({ kinds: [9735], '#p': [pubkey] }); + } + return renderNotifications(filters, types, params, c); }; @@ -81,16 +87,55 @@ async function renderNotifications( const { signal } = c.req.raw; const opts = { signal, limit: params.limit, timeout: Conf.db.timeouts.timelines }; + const zapsRelatedFilter: NostrFilter[] = []; + const events = await store .query(filters, opts) - .then((events) => events.filter((event) => event.pubkey !== pubkey)) + .then((events) => + events.filter((event) => { + if (event.kind === 9735) { + const zappedEventId = event.tags.find(([name]) => name === 'e')?.[1]; + if (zappedEventId) zapsRelatedFilter.push({ kinds: [1], ids: [zappedEventId] }); + const zapSender = event.tags.find(([name]) => name === 'P')?.[1]; + if (zapSender) zapsRelatedFilter.push({ kinds: [0], authors: [zapSender] }); + } + + return event.pubkey !== pubkey; + }) + ) .then((events) => hydrateEvents({ events, store, signal })); if (!events.length) { return c.json([]); } - const notifications = (await Promise.all(events.map((event) => renderNotification(event, { viewerPubkey: pubkey })))) + const zapSendersAndPosts = await store + .query(zapsRelatedFilter, opts) + .then((events) => hydrateEvents({ events, store, signal })); + + const notifications = (await Promise.all(events.map((event) => { + const opts: RenderNotificationOpts = { viewerPubkey: pubkey }; + if (event.kind === 9735) { + const zapRequestString = event?.tags?.find(([name]) => name === 'description')?.[1]; + const zapRequest = n.json().pipe(n.event()).optional().catch(undefined).parse(zapRequestString); + // By getting the pubkey from the zap request we guarantee who is the sender + // some clients don't put the P tag in the zap receipt... + const zapSender = zapRequest?.pubkey; + const zappedPost = event.tags.find(([name]) => name === 'e')?.[1]; + + const amountSchema = z.coerce.number().int().nonnegative().catch(0); + // amount in millisats + const amount = amountSchema.parse(getAmount(event?.tags.find(([name]) => name === 'bolt11')?.[1])); + + opts['zap'] = { + zapSender: zapSendersAndPosts.find(({ pubkey, kind }) => kind === 0 && pubkey === zapSender) ?? zapSender, + zappedPost: zapSendersAndPosts.find(({ id }) => id === zappedPost), + amount, + message: zapRequest?.content, + }; + } + return renderNotification(event, opts); + }))) .filter((notification) => notification && types.has(notification.type)); if (!notifications.length) { From 96e99f38c4f4d1676e77e5a0953e682ba2f2e730 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Mon, 2 Sep 2024 09:53:33 -0300 Subject: [PATCH 10/82] feat(views): render and return zap notification --- src/views/mastodon/notifications.ts | 35 +++++++++++++++++++++++++++-- 1 file changed, 33 insertions(+), 2 deletions(-) diff --git a/src/views/mastodon/notifications.ts b/src/views/mastodon/notifications.ts index 5fc20a2f..f2438ad2 100644 --- a/src/views/mastodon/notifications.ts +++ b/src/views/mastodon/notifications.ts @@ -1,13 +1,19 @@ import { NostrEvent } from '@nostrify/nostrify'; +import { accountFromPubkey, renderAccount } from '@/views/mastodon/accounts.ts'; import { Conf } from '@/config.ts'; import { DittoEvent } from '@/interfaces/DittoEvent.ts'; import { nostrDate } from '@/utils.ts'; -import { accountFromPubkey, renderAccount } from '@/views/mastodon/accounts.ts'; import { renderStatus } from '@/views/mastodon/statuses.ts'; -interface RenderNotificationOpts { +export interface RenderNotificationOpts { viewerPubkey: string; + zap?: { + zapSender?: NostrEvent | NostrEvent['pubkey']; // kind 0 or pubkey + zappedPost?: NostrEvent; + amount?: number; + message?: string; + }; } function renderNotification(event: DittoEvent, opts: RenderNotificationOpts) { @@ -32,6 +38,10 @@ function renderNotification(event: DittoEvent, opts: RenderNotificationOpts) { if (event.kind === 30360 && event.pubkey === Conf.pubkey) { return renderNameGrant(event); } + + if (event.kind === 9735) { + return renderZap(event, opts); + } } async function renderMention(event: DittoEvent, opts: RenderNotificationOpts) { @@ -109,6 +119,27 @@ async function renderNameGrant(event: DittoEvent) { }; } +async function renderZap(event: DittoEvent, opts: RenderNotificationOpts) { + if (!opts.zap?.zapSender) return; + + const { amount = 0, message = '' } = opts.zap; + if (amount < 1) return; + + const account = typeof opts.zap.zapSender !== 'string' + ? await renderAccount(opts.zap.zapSender) + : await accountFromPubkey(opts.zap.zapSender); + + return { + id: notificationId(event), + type: 'ditto:zap', + amount, + message, + created_at: nostrDate(event.created_at).toISOString(), + account, + ...(opts.zap?.zappedPost ? { status: await renderStatus(opts.zap?.zappedPost, opts) } : {}), + }; +} + /** This helps notifications be sorted in the correct order. */ function notificationId({ id, created_at }: NostrEvent): string { return `${created_at}-${id}`; From 1777224f97d22b662dabf6a510ea02a86695bcdf Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Mon, 2 Sep 2024 10:00:23 -0300 Subject: [PATCH 11/82] refactor: rename clean:type-errors to clean:deps --- deno.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deno.json b/deno.json index b90373bf..bd78ae23 100644 --- a/deno.json +++ b/deno.json @@ -19,7 +19,7 @@ "stats:recompute": "deno run -A scripts/stats-recompute.ts", "soapbox": "curl -O https://dl.soapbox.pub/main/soapbox.zip && mkdir -p public && mv soapbox.zip public/ && cd public/ && unzip soapbox.zip && rm soapbox.zip", "trends": "deno run -A scripts/trends.ts", - "clean:type-errors": "deno cache --reload src/app.ts" + "clean:deps": "deno cache --reload src/app.ts" }, "unstable": ["cron", "ffi", "kv", "worker-options"], "exclude": ["./public"], From 7c319c6ff0ceaf4650ffe9b9e9f78bb404ec0462 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Tue, 3 Sep 2024 00:58:35 +0200 Subject: [PATCH 12/82] streaming: don't notify self of own post --- src/controllers/api/streaming.ts | 61 ++++++++++++++++++++++++-------- 1 file changed, 47 insertions(+), 14 deletions(-) diff --git a/src/controllers/api/streaming.ts b/src/controllers/api/streaming.ts index 9a5b5deb..047aa573 100644 --- a/src/controllers/api/streaming.ts +++ b/src/controllers/api/streaming.ts @@ -38,6 +38,25 @@ const streamSchema = z.enum([ type Stream = z.infer; +/** https://docs.joinmastodon.org/methods/streaming/#events-11 */ +interface StreamingEvent { + /** https://docs.joinmastodon.org/methods/streaming/#events */ + event: + | 'update' + | 'delete' + | 'notification' + | 'filters_changed' + | 'conversation' + | 'announcement' + | 'announcement.reaction' + | 'announcement.delete' + | 'status.update' + | 'encrypted_message' + | 'notifications_merged'; + payload: string; + stream: Stream[]; +} + const LIMITER_WINDOW = Time.minutes(5); const LIMITER_LIMIT = 100; @@ -73,18 +92,14 @@ const streamingController: AppController = async (c) => { const policy = pubkey ? new MuteListPolicy(pubkey, await Storages.admin()) : undefined; - function send(name: string, payload: object) { + function send(e: StreamingEvent) { if (socket.readyState === WebSocket.OPEN) { - debug('send', name, JSON.stringify(payload)); - socket.send(JSON.stringify({ - event: name, - payload: JSON.stringify(payload), - stream: [stream], - })); + debug('send', e.event, e.payload); + socket.send(JSON.stringify(e)); } } - async function sub(type: string, filters: NostrFilter[], render: (event: NostrEvent) => Promise) { + async function sub(filters: NostrFilter[], render: (event: NostrEvent) => Promise) { try { for await (const msg of pubsub.req(filters, { signal: controller.signal })) { if (msg[0] === 'EVENT') { @@ -102,7 +117,7 @@ const streamingController: AppController = async (c) => { const result = await render(event); if (result) { - send(type, result); + send(result); } } } @@ -118,19 +133,37 @@ const streamingController: AppController = async (c) => { const topicFilter = await topicToFilter(stream, c.req.query(), pubkey); if (topicFilter) { - sub('update', [topicFilter], async (event) => { + sub([topicFilter], async (event) => { + let payload: object | undefined; + if (event.kind === 1) { - return await renderStatus(event, { viewerPubkey: pubkey }); + payload = await renderStatus(event, { viewerPubkey: pubkey }); } if (event.kind === 6) { - return await renderReblog(event, { viewerPubkey: pubkey }); + payload = await renderReblog(event, { viewerPubkey: pubkey }); + } + + if (payload) { + return { + event: 'update', + payload: JSON.stringify(payload), + stream: [stream], + }; } }); } if (['user', 'user:notification'].includes(stream) && pubkey) { - sub('notification', [{ '#p': [pubkey] }], async (event) => { - return await renderNotification(event, { viewerPubkey: pubkey }); + sub([{ '#p': [pubkey] }], async (event) => { + if (event.pubkey === pubkey) return; // skip own events + const payload = await renderNotification(event, { viewerPubkey: pubkey }); + if (payload) { + return { + event: 'notification', + payload: JSON.stringify(payload), + stream: [stream], + }; + } }); return; } From 6d9d2fd42a1fa39185c0178db16d431d43fa1cbb Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Tue, 3 Sep 2024 22:22:17 -0300 Subject: [PATCH 13/82] fix: get event id from max_id sometimes the 'max_id' format can come as `${created_at}-${id}` so if that's the case, we split by the - (minus) character --- src/schemas/pagination.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/schemas/pagination.ts b/src/schemas/pagination.ts index d7a6cf68..89e3c5f6 100644 --- a/src/schemas/pagination.ts +++ b/src/schemas/pagination.ts @@ -2,7 +2,10 @@ import { z } from 'zod'; /** Schema to parse pagination query params. */ export const paginationSchema = z.object({ - max_id: z.string().optional().catch(undefined), + max_id: z.string().transform((val) => { + if (!val.includes('-')) return val; + return val.split('-')[1]; + }).optional().catch(undefined), min_id: z.string().optional().catch(undefined), since: z.coerce.number().nonnegative().optional().catch(undefined), until: z.coerce.number().nonnegative().optional().catch(undefined), From 486dff83b98c01b5818c632916732ee27df9034e Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Tue, 3 Sep 2024 22:23:15 -0300 Subject: [PATCH 14/82] fix: pass parameters params in ditto:zap notification --- src/controllers/api/notifications.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/controllers/api/notifications.ts b/src/controllers/api/notifications.ts index 00ac9930..864e54d4 100644 --- a/src/controllers/api/notifications.ts +++ b/src/controllers/api/notifications.ts @@ -70,7 +70,7 @@ const notificationsController: AppController = async (c) => { } if (types.has('ditto:zap')) { - filters.push({ kinds: [9735], '#p': [pubkey] }); + filters.push({ kinds: [9735], '#p': [pubkey], ...params }); } return renderNotifications(filters, types, params, c); From a8900b3217b0cec1995992ea5bc6f48b5204834c Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Thu, 5 Sep 2024 18:43:02 -0300 Subject: [PATCH 15/82] fix(zap notification): put kind 9735 in the first filter --- src/controllers/api/notifications.ts | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/controllers/api/notifications.ts b/src/controllers/api/notifications.ts index 864e54d4..1b9c746a 100644 --- a/src/controllers/api/notifications.ts +++ b/src/controllers/api/notifications.ts @@ -52,6 +52,9 @@ const notificationsController: AppController = async (c) => { if (types.has('favourite') || types.has('pleroma:emoji_reaction')) { kinds.add(7); } + if (types.has('ditto:zap')) { + kinds.add(9735); + } const filter: NostrFilter = { kinds: [...kinds], @@ -69,10 +72,6 @@ const notificationsController: AppController = async (c) => { filters.push({ kinds: [30360], authors: [Conf.pubkey], '#p': [pubkey], ...params }); } - if (types.has('ditto:zap')) { - filters.push({ kinds: [9735], '#p': [pubkey], ...params }); - } - return renderNotifications(filters, types, params, c); }; From 5454942a2c14e2ae6947fe3ace0f36bec4119234 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 7 Sep 2024 08:52:02 -0500 Subject: [PATCH 16/82] Update Prometheus metrics to conform to best practices --- src/controllers/metrics.ts | 3 +- src/controllers/nostr/relay.ts | 8 ++--- src/db/KyselyLogger.ts | 6 ++-- src/firehose.ts | 4 +-- src/metrics.ts | 53 +++++++++++++---------------- src/middleware/metricsMiddleware.ts | 6 ++-- src/pipeline.ts | 6 ++-- src/storages/EventsDB.ts | 4 +-- 8 files changed, 42 insertions(+), 48 deletions(-) diff --git a/src/controllers/metrics.ts b/src/controllers/metrics.ts index e85294c0..567a3170 100644 --- a/src/controllers/metrics.ts +++ b/src/controllers/metrics.ts @@ -2,12 +2,11 @@ import { register } from 'prom-client'; import { AppController } from '@/app.ts'; import { DittoDB } from '@/db/DittoDB.ts'; -import { dbAvailableConnectionsGauge, dbPoolSizeGauge } from '@/metrics.ts'; +import { dbAvailableConnectionsGauge } from '@/metrics.ts'; /** Prometheus/OpenMetrics controller. */ export const metricsController: AppController = async (c) => { // Update some metrics at request time. - dbPoolSizeGauge.set(DittoDB.poolSize); dbAvailableConnectionsGauge.set(DittoDB.availableConnections); const metrics = await register.metrics(); diff --git a/src/controllers/nostr/relay.ts b/src/controllers/nostr/relay.ts index 583fd153..9f47b382 100644 --- a/src/controllers/nostr/relay.ts +++ b/src/controllers/nostr/relay.ts @@ -12,7 +12,7 @@ import { import { AppController } from '@/app.ts'; import { Conf } from '@/config.ts'; import { relayInfoController } from '@/controllers/nostr/relay-info.ts'; -import { relayConnectionsGauge, relayEventCounter, relayMessageCounter } from '@/metrics.ts'; +import { relayConnectionsGauge, relayEventsCounter, relayMessagesCounter } from '@/metrics.ts'; import * as pipeline from '@/pipeline.ts'; import { RelayError } from '@/RelayError.ts'; import { Storages } from '@/storages.ts'; @@ -54,10 +54,10 @@ function connectStream(socket: WebSocket, ip: string | undefined) { const result = n.json().pipe(n.clientMsg()).safeParse(e.data); if (result.success) { - relayMessageCounter.inc({ verb: result.data[0] }); + relayMessagesCounter.inc({ verb: result.data[0] }); handleMsg(result.data); } else { - relayMessageCounter.inc(); + relayMessagesCounter.inc(); send(['NOTICE', 'Invalid message.']); } }; @@ -130,7 +130,7 @@ function connectStream(socket: WebSocket, ip: string | undefined) { /** Handle EVENT. Store the event. */ async function handleEvent([_, event]: NostrClientEVENT): Promise { - relayEventCounter.inc({ kind: event.kind.toString() }); + relayEventsCounter.inc({ kind: event.kind.toString() }); try { // This will store it (if eligible) and run other side-effects. await pipeline.handleEvent(event, AbortSignal.timeout(1000)); diff --git a/src/db/KyselyLogger.ts b/src/db/KyselyLogger.ts index 3b5b4398..72167e21 100644 --- a/src/db/KyselyLogger.ts +++ b/src/db/KyselyLogger.ts @@ -1,6 +1,6 @@ import { Stickynotes } from '@soapbox/stickynotes'; import { Logger } from 'kysely'; -import { dbQueryCounter, dbQueryTimeHistogram } from '@/metrics.ts'; +import { dbQueriesCounter, dbQueryDurationHistogram } from '@/metrics.ts'; /** Log the SQL for queries. */ export const KyselyLogger: Logger = (event) => { @@ -9,8 +9,8 @@ export const KyselyLogger: Logger = (event) => { const { query, queryDurationMillis } = event; const { sql, parameters } = query; - dbQueryCounter.inc(); - dbQueryTimeHistogram.observe(queryDurationMillis); + dbQueriesCounter.inc(); + dbQueryDurationHistogram.observe(queryDurationMillis); console.debug( sql, diff --git a/src/firehose.ts b/src/firehose.ts index 86d19f74..85e3dc89 100644 --- a/src/firehose.ts +++ b/src/firehose.ts @@ -2,7 +2,7 @@ import { Semaphore } from '@lambdalisue/async'; import { Stickynotes } from '@soapbox/stickynotes'; import { Conf } from '@/config.ts'; -import { firehoseEventCounter } from '@/metrics.ts'; +import { firehoseEventsCounter } from '@/metrics.ts'; import { Storages } from '@/storages.ts'; import { nostrNow } from '@/utils.ts'; @@ -23,7 +23,7 @@ export async function startFirehose(): Promise { if (msg[0] === 'EVENT') { const event = msg[2]; console.debug(`NostrEvent<${event.kind}> ${event.id}`); - firehoseEventCounter.inc({ kind: event.kind }); + firehoseEventsCounter.inc({ kind: event.kind }); sem.lock(async () => { try { diff --git a/src/metrics.ts b/src/metrics.ts index c1fb8238..ee73d083 100644 --- a/src/metrics.ts +++ b/src/metrics.ts @@ -1,86 +1,81 @@ import { Counter, Gauge, Histogram } from 'prom-client'; -export const httpRequestCounter = new Counter({ - name: 'http_requests_total', +export const httpRequestsCounter = new Counter({ + name: 'ditto_http_requests_total', help: 'Total number of HTTP requests', labelNames: ['method'], }); -export const httpResponseCounter = new Counter({ - name: 'http_responses_total', +export const httpResponsesCounter = new Counter({ + name: 'ditto_http_responses_total', help: 'Total number of HTTP responses', labelNames: ['method', 'path', 'status'], }); export const streamingConnectionsGauge = new Gauge({ - name: 'streaming_connections', + name: 'ditto_streaming_connections', help: 'Number of active connections to the streaming API', }); export const fetchCounter = new Counter({ - name: 'fetch_total', + name: 'ditto_fetch_total', help: 'Total number of fetch requests', labelNames: ['method'], }); -export const firehoseEventCounter = new Counter({ - name: 'firehose_events_total', +export const firehoseEventsCounter = new Counter({ + name: 'ditto_firehose_events_total', help: 'Total number of Nostr events processed by the firehose', labelNames: ['kind'], }); -export const pipelineEventCounter = new Counter({ - name: 'pipeline_events_total', +export const pipelineEventsCounter = new Counter({ + name: 'ditto_pipeline_events_total', help: 'Total number of Nostr events processed by the pipeline', labelNames: ['kind'], }); -export const policyEventCounter = new Counter({ - name: 'policy_events_total', +export const policyEventsCounter = new Counter({ + name: 'ditto_policy_events_total', help: 'Total number of policy OK responses', labelNames: ['ok'], }); -export const relayEventCounter = new Counter({ - name: 'relay_events_total', +export const relayEventsCounter = new Counter({ + name: 'ditto_relay_events_total', help: 'Total number of EVENT messages processed by the relay', labelNames: ['kind'], }); -export const relayMessageCounter = new Counter({ - name: 'relay_messages_total', +export const relayMessagesCounter = new Counter({ + name: 'ditto_relay_messages_total', help: 'Total number of Nostr messages processed by the relay', labelNames: ['verb'], }); export const relayConnectionsGauge = new Gauge({ - name: 'relay_connections', + name: 'ditto_relay_connections', help: 'Number of active connections to the relay', }); -export const dbQueryCounter = new Counter({ - name: 'db_query_total', +export const dbQueriesCounter = new Counter({ + name: 'ditto_db_queries_total', help: 'Total number of database queries', labelNames: ['kind'], }); -export const dbEventCounter = new Counter({ - name: 'db_events_total', +export const dbEventsCounter = new Counter({ + name: 'ditto_db_events_total', help: 'Total number of database inserts', labelNames: ['kind'], }); -export const dbPoolSizeGauge = new Gauge({ - name: 'db_pool_size', - help: 'Number of connections in the database pool', -}); - export const dbAvailableConnectionsGauge = new Gauge({ - name: 'db_available_connections', + name: 'ditto_db_available_connections', help: 'Number of available connections in the database pool', }); -export const dbQueryTimeHistogram = new Histogram({ - name: 'db_query_duration_ms', +export const dbQueryDurationHistogram = new Histogram({ + name: 'ditto_db_query_duration_ms', help: 'Duration of database queries', }); diff --git a/src/middleware/metricsMiddleware.ts b/src/middleware/metricsMiddleware.ts index 6cf0e6de..e8a30972 100644 --- a/src/middleware/metricsMiddleware.ts +++ b/src/middleware/metricsMiddleware.ts @@ -1,12 +1,12 @@ import { MiddlewareHandler } from '@hono/hono'; -import { httpRequestCounter, httpResponseCounter } from '@/metrics.ts'; +import { httpRequestsCounter, httpResponsesCounter } from '@/metrics.ts'; /** Prometheus metrics middleware that tracks HTTP requests by methods and responses by status code. */ export const metricsMiddleware: MiddlewareHandler = async (c, next) => { // HTTP Request. const { method } = c.req; - httpRequestCounter.inc({ method }); + httpRequestsCounter.inc({ method }); // Wait for other handlers to run. await next(); @@ -16,5 +16,5 @@ export const metricsMiddleware: MiddlewareHandler = async (c, next) => { // Get a parameterized path name like `/posts/:id` instead of `/posts/1234`. // Tries to find actual route names first before falling back on potential middleware handlers like `app.use('*')`. const path = c.req.matchedRoutes.find((r) => r.method !== 'ALL')?.path ?? c.req.routePath; - httpResponseCounter.inc({ method, status, path }); + httpResponsesCounter.inc({ method, status, path }); }; diff --git a/src/pipeline.ts b/src/pipeline.ts index 2fb18649..15c1ef34 100644 --- a/src/pipeline.ts +++ b/src/pipeline.ts @@ -8,7 +8,7 @@ import { Conf } from '@/config.ts'; import { DittoDB } from '@/db/DittoDB.ts'; import { deleteAttachedMedia } from '@/db/unattached-media.ts'; import { DittoEvent } from '@/interfaces/DittoEvent.ts'; -import { pipelineEventCounter, policyEventCounter } from '@/metrics.ts'; +import { pipelineEventsCounter, policyEventsCounter } from '@/metrics.ts'; import { RelayError } from '@/RelayError.ts'; import { AdminSigner } from '@/signers/AdminSigner.ts'; import { hydrateEvents } from '@/storages/hydrate.ts'; @@ -40,7 +40,7 @@ async function handleEvent(event: DittoEvent, signal: AbortSignal): Promise ${event.id}`); - pipelineEventCounter.inc({ kind: event.kind }); + pipelineEventsCounter.inc({ kind: event.kind }); if (event.kind !== 24133) { await policyFilter(event); @@ -71,7 +71,7 @@ async function policyFilter(event: NostrEvent): Promise { try { const result = await policyWorker.call(event); - policyEventCounter.inc({ ok: String(result[2]) }); + policyEventsCounter.inc({ ok: String(result[2]) }); debug(JSON.stringify(result)); RelayError.assert(result); } catch (e) { diff --git a/src/storages/EventsDB.ts b/src/storages/EventsDB.ts index 07be8067..7cbd9c22 100644 --- a/src/storages/EventsDB.ts +++ b/src/storages/EventsDB.ts @@ -17,7 +17,7 @@ import { nip27 } from 'nostr-tools'; import { Conf } from '@/config.ts'; import { DittoDatabase } from '@/db/DittoDB.ts'; -import { dbEventCounter } from '@/metrics.ts'; +import { dbEventsCounter } from '@/metrics.ts'; import { RelayError } from '@/RelayError.ts'; import { purifyEvent } from '@/storages/hydrate.ts'; import { isNostrId, isURL } from '@/utils.ts'; @@ -73,7 +73,7 @@ class EventsDB implements NStore { async event(event: NostrEvent, opts: { signal?: AbortSignal; timeout?: number } = {}): Promise { event = purifyEvent(event); this.console.debug('EVENT', JSON.stringify(event)); - dbEventCounter.inc({ kind: event.kind }); + dbEventsCounter.inc({ kind: event.kind }); if (await this.isDeletedAdmin(event)) { throw new RelayError('blocked', 'event deleted by admin'); From 4a578528f587cc5f90163ac757b6bb1a15850155 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 7 Sep 2024 09:00:18 -0500 Subject: [PATCH 17/82] Add back pool size gauge --- src/controllers/metrics.ts | 3 ++- src/metrics.ts | 5 +++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/controllers/metrics.ts b/src/controllers/metrics.ts index 567a3170..e85294c0 100644 --- a/src/controllers/metrics.ts +++ b/src/controllers/metrics.ts @@ -2,11 +2,12 @@ import { register } from 'prom-client'; import { AppController } from '@/app.ts'; import { DittoDB } from '@/db/DittoDB.ts'; -import { dbAvailableConnectionsGauge } from '@/metrics.ts'; +import { dbAvailableConnectionsGauge, dbPoolSizeGauge } from '@/metrics.ts'; /** Prometheus/OpenMetrics controller. */ export const metricsController: AppController = async (c) => { // Update some metrics at request time. + dbPoolSizeGauge.set(DittoDB.poolSize); dbAvailableConnectionsGauge.set(DittoDB.availableConnections); const metrics = await register.metrics(); diff --git a/src/metrics.ts b/src/metrics.ts index ee73d083..ac1db2ee 100644 --- a/src/metrics.ts +++ b/src/metrics.ts @@ -70,6 +70,11 @@ export const dbEventsCounter = new Counter({ labelNames: ['kind'], }); +export const dbPoolSizeGauge = new Gauge({ + name: 'ditto_db_pool_size', + help: 'Number of connections in the database pool', +}); + export const dbAvailableConnectionsGauge = new Gauge({ name: 'ditto_db_available_connections', help: 'Number of available connections in the database pool', From 537de854214afb40222a9bf2c1b85adf75abd51f Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 7 Sep 2024 09:02:14 -0500 Subject: [PATCH 18/82] Add grafana dashboard.json --- grafana/Ditto-Dashboard.json | 2117 ++++++++++++++++++++++++++++++++++ 1 file changed, 2117 insertions(+) create mode 100644 grafana/Ditto-Dashboard.json diff --git a/grafana/Ditto-Dashboard.json b/grafana/Ditto-Dashboard.json new file mode 100644 index 00000000..222b727c --- /dev/null +++ b/grafana/Ditto-Dashboard.json @@ -0,0 +1,2117 @@ +{ + "__inputs": [ + { + "name": "DS_PROMETHEUS", + "label": "prometheus", + "description": "", + "type": "datasource", + "pluginId": "prometheus", + "pluginName": "Prometheus" + }, + { + "name": "DS_DITTO-PG", + "label": "ditto-pg", + "description": "", + "type": "datasource", + "pluginId": "grafana-postgresql-datasource", + "pluginName": "PostgreSQL" + } + ], + "__elements": {}, + "__requires": [ + { + "type": "panel", + "id": "gauge", + "name": "Gauge", + "version": "" + }, + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "11.2.0" + }, + { + "type": "datasource", + "id": "grafana-postgresql-datasource", + "name": "PostgreSQL", + "version": "1.0.0" + }, + { + "type": "panel", + "id": "piechart", + "name": "Pie chart", + "version": "" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "1.0.0" + }, + { + "type": "panel", + "id": "table", + "name": "Table", + "version": "" + }, + { + "type": "panel", + "id": "timeseries", + "name": "Time series", + "version": "" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "Ditto application performance", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": null, + "links": [], + "panels": [ + { + "datasource": { + "default": false, + "type": "prometheus", + "uid": "${prometheus}" + }, + "description": "Responses sent back to users from the server. Organized by status code.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 25, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "Server Errors" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "semi-dark-red", + "mode": "shades" + } + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "Client Errors" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "dark-blue", + "mode": "shades" + } + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "Redirects" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "semi-dark-yellow", + "mode": "shades" + } + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "Success" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "semi-dark-green", + "mode": "shades" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 10, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": false, + "width": 225 + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "increase(ditto_http_responses_total{status=~\"5..\",path!=\"/relay\"}[$__rate_interval])", + "hide": false, + "instant": false, + "legendFormat": "{{method}} {{path}} ({{status}})", + "range": true, + "refId": "Server Errors" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheus}" + }, + "editorMode": "code", + "expr": "increase(ditto_http_responses_total{status=~\"4..\",path!=\"/relay\"}[$__rate_interval])", + "hide": false, + "instant": false, + "legendFormat": "{{method}} {{path}} ({{status}})", + "range": true, + "refId": "Client Errors" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "increase(ditto_http_responses_total{status=~\"3..\",path!=\"/relay\"}[$__rate_interval])", + "hide": false, + "instant": false, + "legendFormat": "{{method}} {{path}} ({{status}})", + "range": true, + "refId": "Redirects" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheus}" + }, + "editorMode": "code", + "expr": "increase(ditto_http_responses_total{status=~\"2..\",path!=\"/relay\"}[$__rate_interval])", + "instant": false, + "interval": "", + "legendFormat": "{{method}} {{path}} ({{status}})", + "range": true, + "refId": "Success" + } + ], + "title": "HTTP Responses", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Number of idle database connections available to the server. Higher is better. At 0, the site stops working.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-RdYlGr" + }, + "mappings": [], + "thresholds": { + "mode": "percentage", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "red", + "value": 0 + }, + { + "color": "#EAB839", + "value": 25 + }, + { + "color": "green", + "value": 50 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 0, + "y": 10 + }, + "id": 8, + "options": { + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "sizing": "auto" + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${prometheus}" + }, + "editorMode": "code", + "expr": "ditto_db_available_connections", + "instant": false, + "legendFormat": "Connections", + "range": true, + "refId": "A" + } + ], + "title": "Database Available Connections", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "How hard the database is working on a per-query basis. It's query time divided by number of queries. Lower is better.", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "red", + "mode": "continuous-GrYlRd", + "seriesBy": "last" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "scheme", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 6, + "y": 10 + }, + "id": 11, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${prometheus}" + }, + "editorMode": "code", + "expr": "rate(ditto_db_query_duration_ms_sum[$__rate_interval]) / rate(db_query_total[$__rate_interval])", + "instant": false, + "interval": "", + "legendFormat": "Effort", + "range": true, + "refId": "A" + } + ], + "title": "Database Effort", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Number of individual database calls.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 18, + "y": 10 + }, + "id": 6, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${prometheus}" + }, + "editorMode": "code", + "expr": "increase(ditto_db_queries_total[$__rate_interval])", + "instant": false, + "legendFormat": "Queries", + "range": true, + "refId": "A" + } + ], + "title": "Database Queries", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Number of Nostr events that are accepted or rejected by the custom policy script.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Accept" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Reject" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 0, + "y": 18 + }, + "id": 12, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${prometheus}" + }, + "editorMode": "code", + "expr": "increase(ditto_policy_events_total{ok=\"true\"}[$__rate_interval])", + "instant": false, + "legendFormat": "Accept", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "increase(ditto_policy_events_total{ok=\"false\"}[$__rate_interval])", + "hide": false, + "instant": false, + "legendFormat": "Reject", + "range": true, + "refId": "B" + } + ], + "title": "Policy Events", + "type": "timeseries" + }, + { + "datasource": { + "default": false, + "type": "prometheus", + "uid": "${prometheus}" + }, + "description": "Total number of Nostr clients currently connected to the Nostr relay.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 6, + "y": 18 + }, + "id": 2, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "ditto_relay_connections", + "instant": false, + "legendFormat": "Connections", + "range": true, + "refId": "A" + } + ], + "title": "Relay Connections", + "type": "timeseries" + }, + { + "datasource": { + "default": false, + "type": "prometheus", + "uid": "${prometheus}" + }, + "description": "Sum duration of all queries in milliseconds.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 18, + "y": 18 + }, + "id": 7, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "increase(ditto_db_query_duration_ms_sum[$__rate_interval])", + "instant": false, + "legendFormat": "Time", + "range": true, + "refId": "A" + } + ], + "title": "Database Query Time", + "type": "timeseries" + }, + { + "datasource": { + "default": false, + "type": "grafana-postgresql-datasource", + "uid": "${postgres}" + }, + "description": "SQL queries ranked by total time.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "super-light-green", + "value": 100 + }, + { + "color": "light-orange", + "value": 500 + }, + { + "color": "red", + "value": 1000 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "query" + }, + "properties": [ + { + "id": "displayName", + "value": "Query" + }, + { + "id": "custom.inspect", + "value": true + }, + { + "id": "custom.minWidth", + "value": 500 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "total_time" + }, + "properties": [ + { + "id": "displayName", + "value": "Total Time" + }, + { + "id": "unit", + "value": "ms" + }, + { + "id": "decimals", + "value": 1 + }, + { + "id": "custom.hidden", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "average_time" + }, + "properties": [ + { + "id": "displayName", + "value": "Average Time" + }, + { + "id": "unit", + "value": "ms" + }, + { + "id": "custom.cellOptions", + "value": { + "type": "color-text" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "calls" + }, + "properties": [ + { + "id": "displayName", + "value": "Calls" + }, + { + "id": "unit", + "value": "short" + }, + { + "id": "decimals", + "value": 0 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "total_percent" + }, + "properties": [ + { + "id": "displayName", + "value": "Total Percent" + }, + { + "id": "unit", + "value": "percentunit" + }, + { + "id": "decimals", + "value": 0 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "max_time" + }, + "properties": [ + { + "id": "displayName", + "value": "Max Time" + }, + { + "id": "unit", + "value": "ms" + }, + { + "id": "custom.cellOptions", + "value": { + "type": "color-text" + } + } + ] + } + ] + }, + "gridPos": { + "h": 11, + "w": 15, + "x": 0, + "y": 26 + }, + "id": 13, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "enablePagination": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": true + }, + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "Total Percent" + } + ] + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "${DS_DITTO-PG}" + }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "WITH query_stats AS (\n SELECT\n LEFT(query, 10000) AS query,\n (total_plan_time + total_exec_time) AS total_time,\n (mean_plan_time + mean_exec_time) AS average_time,\n (max_plan_time + max_exec_time) AS max_time,\n calls\n FROM\n pg_stat_statements\n WHERE\n calls > 0\n)\nSELECT\n query,\n total_time,\n average_time,\n max_time,\n calls,\n total_time / (SELECT SUM(total_time) FROM query_stats) AS total_percent\nFROM\n query_stats\nORDER BY\n total_time DESC\nLIMIT 100;\n", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "pg_stat_statements" + } + ], + "title": "Query Performance", + "type": "table" + }, + { + "datasource": { + "name": "${postgres}", + "type": "grafana-postgresql-datasource", + "uid": "${postgres}" + }, + "description": "Index hit rate. Higher is better.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "table" + }, + "properties": [ + { + "id": "displayName", + "value": "Table" + }, + { + "id": "custom.filterable", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "index" + }, + "properties": [ + { + "id": "displayName", + "value": "Index" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "percentage_used" + }, + "properties": [ + { + "id": "displayName", + "value": "Percentage Used" + }, + { + "id": "unit", + "value": "percent" + }, + { + "id": "custom.cellOptions", + "value": { + "mode": "lcd", + "type": "gauge", + "valueDisplayMode": "color" + } + }, + { + "id": "color", + "value": { + "mode": "continuous-RdYlGr" + } + } + ] + } + ] + }, + "gridPos": { + "h": 11, + "w": 9, + "x": 15, + "y": 26 + }, + "id": 14, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "${DS_DITTO-PG}" + }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT \n relname AS table,\n indexrelname AS index,\n (idx_blks_hit * 100) / (CASE WHEN idx_blks_hit + idx_blks_read = 0 THEN 1 ELSE idx_blks_hit + idx_blks_read END) as percentage_used\nFROM \n pg_statio_user_indexes\nWHERE \n schemaname = 'public'\nORDER BY\n percentage_used DESC;", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Index Uses", + "type": "table" + }, + { + "datasource": { + "name": "${postgres}", + "type": "grafana-postgresql-datasource", + "uid": "${postgres}" + }, + "description": "Size of database tables.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "mappings": [], + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 7, + "x": 0, + "y": 37 + }, + "id": 16, + "options": { + "displayLabels": [ + "name" + ], + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": false, + "values": [ + "value" + ] + }, + "pieType": "pie", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "${DS_DITTO-PG}" + }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT\n c.relname AS table,\n pg_table_size(c.oid) AS size\nFROM\n pg_class c\n LEFT JOIN pg_namespace n ON (n.oid = c.relnamespace)\nWHERE\n n.nspname NOT IN ('pg_catalog', 'information_schema')\n AND n.nspname !~ '^pg_toast'\n AND c.relkind IN ('r', 'm')\nORDER BY\n pg_table_size(c.oid) DESC;", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Table Size", + "transformations": [ + { + "id": "rowsToFields", + "options": {} + } + ], + "type": "piechart" + }, + { + "datasource": { + "name": "${postgres}", + "type": "grafana-postgresql-datasource", + "uid": "${postgres}" + }, + "description": "Size of all indexes in the database.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "mappings": [], + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 7, + "y": 37 + }, + "id": 17, + "options": { + "displayLabels": [ + "name" + ], + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "right", + "showLegend": false, + "values": [] + }, + "pieType": "pie", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "${DS_DITTO-PG}" + }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT\n c.relname AS name,\n sum(c.relpages :: bigint * 8192) :: bigint AS size\nFROM\n pg_class c\n LEFT JOIN pg_namespace n ON (n.oid = c.relnamespace)\nWHERE\n n.nspname NOT IN ('pg_catalog', 'information_schema')\n AND n.nspname !~ '^pg_toast'\n AND c.relkind = 'i'\nGROUP BY\n (n.nspname, c.relname)\nORDER BY\n sum(c.relpages) DESC;", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Index Size", + "transformations": [ + { + "id": "rowsToFields", + "options": {} + } + ], + "type": "piechart" + }, + { + "datasource": { + "name": "${postgres}", + "type": "grafana-postgresql-datasource", + "uid": "${postgres}" + }, + "description": "Indexes that are not used or very rarely used.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "table" + }, + "properties": [ + { + "id": "displayName", + "value": "Table" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "index" + }, + "properties": [ + { + "id": "displayName", + "value": "Index" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "index_size" + }, + "properties": [ + { + "id": "displayName", + "value": "Size" + }, + { + "id": "unit", + "value": "decbytes" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "index_scans" + }, + "properties": [ + { + "id": "displayName", + "value": "Scans" + }, + { + "id": "unit", + "value": "short" + } + ] + } + ] + }, + "gridPos": { + "h": 9, + "w": 9, + "x": 15, + "y": 37 + }, + "id": 18, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "${DS_DITTO-PG}" + }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT\n relname AS table,\n indexrelname AS index,\n pg_relation_size(i.indexrelid) AS index_size,\n idx_scan as index_scans\nFROM\n pg_stat_user_indexes ui\n JOIN pg_index i ON ui.indexrelid = i.indexrelid\nWHERE\n NOT indisunique\n AND idx_scan < 50\n AND pg_relation_size(relid) > 5 * 8192\nORDER BY\n pg_relation_size(i.indexrelid) / nullif(idx_scan, 0) DESC NULLS FIRST,\n pg_relation_size(i.indexrelid) DESC;", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Unused Indexes", + "type": "table" + }, + { + "datasource": { + "default": false, + "type": "prometheus", + "uid": "${prometheus}" + }, + "description": "Nostr messages sent to the relay over WebSocket.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "{instance=\"localhost:4036\", job=\"ditto\"}" + }, + "properties": [ + { + "id": "displayName", + "value": "Other" + }, + { + "id": "color", + "value": { + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "REQ" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "purple", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "EVENT" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-blue", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "CLOSE" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "dark-yellow", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 12, + "w": 24, + "x": 0, + "y": 46 + }, + "id": 9, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "increase(ditto_relay_messages_total[$__rate_interval])", + "format": "time_series", + "instant": false, + "legendFormat": "{{verb}}", + "range": true, + "refId": "A" + } + ], + "title": "Relay Messages", + "type": "timeseries" + }, + { + "datasource": { + "default": false, + "type": "prometheus", + "uid": "${prometheus}" + }, + "description": "Nostr events from all sources. Organized by kind.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Kind 0" + }, + "properties": [ + { + "id": "links", + "value": [ + { + "targetBlank": true, + "title": "NIP-01", + "url": "https://github.com/nostr-protocol/nips/blob/master/01.md" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Kind 1" + }, + "properties": [ + { + "id": "links", + "value": [ + { + "targetBlank": true, + "title": "NIP-01", + "url": "https://github.com/nostr-protocol/nips/blob/master/01.md" + } + ] + } + ] + } + ] + }, + "gridPos": { + "h": 12, + "w": 24, + "x": 0, + "y": 58 + }, + "id": 5, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "increase(ditto_pipeline_events_total[$__rate_interval])", + "instant": false, + "legendFormat": "Kind {{kind}}", + "range": true, + "refId": "A" + } + ], + "title": "Pipeline Events", + "type": "timeseries" + }, + { + "datasource": { + "default": false, + "type": "prometheus", + "uid": "${prometheus}" + }, + "description": "Rate of events being processed by different parts of the application.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 24, + "x": 0, + "y": 70 + }, + "id": 19, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(increase(ditto_pipeline_events_total[$__rate_interval]))", + "instant": false, + "legendFormat": "Pipeline Events", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheus}" + }, + "editorMode": "code", + "expr": "sum(increase(ditto_relay_events_total[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "Relay Events", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(increase(ditto_firehose_events_total[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "Firehose Events", + "range": true, + "refId": "C" + } + ], + "title": "Event Sources", + "type": "timeseries" + } + ], + "refresh": "5s", + "schemaVersion": 39, + "tags": [], + "templating": { + "list": [ + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "cdtcs576ar7cwb" + }, + "filters": [], + "hide": 0, + "name": "Filters", + "skipUrlSync": false, + "type": "adhoc" + }, + { + "current": {}, + "description": "Prometheus datasource", + "hide": 0, + "includeAll": false, + "label": "Prometheus", + "multi": false, + "name": "prometheus", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "current": {}, + "description": "PostgreSQL datasource", + "hide": 0, + "includeAll": false, + "label": "Postgres", + "multi": false, + "name": "postgres", + "options": [], + "query": "grafana-postgresql-datasource", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + } + ] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "Ditto", + "uid": "ddps3ap51fv28d", + "version": 59, + "weekStart": "" +} \ No newline at end of file From ddba16551aab5d1c2549942577f01620f9036a9d Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 7 Sep 2024 09:41:29 -0500 Subject: [PATCH 19/82] Fix rendering mentions inside of URLs --- src/controllers/api/statuses.ts | 30 +++++++++++++++++------------- src/utils/note.test.ts | 16 ++++++++++++++++ 2 files changed, 33 insertions(+), 13 deletions(-) diff --git a/src/controllers/api/statuses.ts b/src/controllers/api/statuses.ts index 621e26a8..282698e3 100644 --- a/src/controllers/api/statuses.ts +++ b/src/controllers/api/statuses.ts @@ -123,21 +123,25 @@ const createStatusController: AppController = async (c) => { const pubkeys = new Set(); - const content = await asyncReplaceAll(data.status ?? '', /@([\w@+._]+)/g, async (match, username) => { - const pubkey = await lookupPubkey(username); - if (!pubkey) return match; + const content = await asyncReplaceAll( + data.status ?? '', + /(? { + const pubkey = await lookupPubkey(username); + if (!pubkey) return match; - // Content addressing (default) - if (!data.to) { - pubkeys.add(pubkey); - } + // Content addressing (default) + if (!data.to) { + pubkeys.add(pubkey); + } - try { - return `nostr:${nip19.npubEncode(pubkey)}`; - } catch { - return match; - } - }); + try { + return `nostr:${nip19.npubEncode(pubkey)}`; + } catch { + return match; + } + }, + ); // Explicit addressing for (const to of data.to ?? []) { diff --git a/src/utils/note.test.ts b/src/utils/note.test.ts index 39be190b..e8d17e89 100644 --- a/src/utils/note.test.ts +++ b/src/utils/note.test.ts @@ -49,6 +49,22 @@ Deno.test('parseNoteContent renders empty for non-profile nostr URIs', () => { assertEquals(html, ''); }); +Deno.test("parseNoteContent doesn't fuck up links to my own post", () => { + const { html } = parseNoteContent( + 'Check this post: https://gleasonator.dev/@alex@gleasonator.dev/posts/a8badb480d88f9e7b6a090342279ef47ed0e0a3989ed85f898dfedc6be94225f', + [{ + id: '0461fcbecc4c3374439932d6b8f11269ccdb7cc973ad7a50ae362db135a474dd', + username: 'alex', + acct: 'alex@gleasonator.dev', + url: 'https://gleasonator.dev/@alex', + }], + ); + assertEquals( + html, + 'Check this post: https://gleasonator.dev/@alex@gleasonator.dev/posts/a8badb480d88f9e7b6a090342279ef47ed0e0a3989ed85f898dfedc6be94225f', + ); +}); + Deno.test('getMediaLinks', () => { const links = [ { href: 'https://example.com/image.png' }, From 8efd6fbb20403159bcb63b0493f1ff8eed6d3e98 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 7 Sep 2024 10:24:56 -0500 Subject: [PATCH 20/82] Remove unattached_media table, replace with LRUCache, fix media upload order problem --- src/DittoUploads.ts | 17 ++++ src/controllers/api/media.ts | 23 +++-- src/controllers/api/statuses.ts | 30 ++++--- src/db/DittoTables.ts | 9 -- src/db/migrations/031_rm_unattached_media.ts | 34 ++++++++ src/db/unattached-media.ts | 88 -------------------- src/pipeline.ts | 10 --- src/utils/upload.ts | 18 ++-- src/views/mastodon/attachments.ts | 4 +- src/views/mastodon/statuses.ts | 6 +- 10 files changed, 100 insertions(+), 139 deletions(-) create mode 100644 src/DittoUploads.ts create mode 100644 src/db/migrations/031_rm_unattached_media.ts delete mode 100644 src/db/unattached-media.ts diff --git a/src/DittoUploads.ts b/src/DittoUploads.ts new file mode 100644 index 00000000..044d3585 --- /dev/null +++ b/src/DittoUploads.ts @@ -0,0 +1,17 @@ +import { LRUCache } from 'lru-cache'; + +import { Time } from '@/utils/time.ts'; + +export interface DittoUpload { + id: string; + pubkey: string; + description?: string; + url: string; + tags: string[][]; + uploadedAt: Date; +} + +export const dittoUploads = new LRUCache({ + max: 1000, + ttl: Time.minutes(15), +}); diff --git a/src/controllers/api/media.ts b/src/controllers/api/media.ts index 3fa3ee68..9dc2de27 100644 --- a/src/controllers/api/media.ts +++ b/src/controllers/api/media.ts @@ -5,7 +5,7 @@ import { fileSchema } from '@/schema.ts'; import { parseBody } from '@/utils/api.ts'; import { renderAttachment } from '@/views/mastodon/attachments.ts'; import { uploadFile } from '@/utils/upload.ts'; -import { setMediaDescription } from '@/db/unattached-media.ts'; +import { dittoUploads } from '@/DittoUploads.ts'; const mediaBodySchema = z.object({ file: fileSchema, @@ -39,19 +39,24 @@ const mediaController: AppController = async (c) => { const updateMediaController: AppController = async (c) => { const result = mediaUpdateSchema.safeParse(await parseBody(c.req.raw)); + if (!result.success) { return c.json({ error: 'Bad request.', schema: result.error }, 422); } - try { - const { description } = result.data; - if (!await setMediaDescription(c.req.param('id'), description)) { - return c.json({ error: 'File with specified ID not found.' }, 404); - } - } catch (e) { - console.error(e); - return c.json({ error: 'Failed to set media description.' }, 500); + + const id = c.req.param('id'); + const { description } = result.data; + const upload = dittoUploads.get(id); + + if (!upload) { + return c.json({ error: 'File with specified ID not found.' }, 404); } + dittoUploads.set(id, { + ...upload, + description, + }); + return c.json({ message: 'ok' }, 200); }; diff --git a/src/controllers/api/statuses.ts b/src/controllers/api/statuses.ts index 282698e3..47adc572 100644 --- a/src/controllers/api/statuses.ts +++ b/src/controllers/api/statuses.ts @@ -1,3 +1,4 @@ +import { HTTPException } from '@hono/hono/http-exception'; import { NostrEvent, NSchema as n } from '@nostrify/nostrify'; import ISO6391 from 'iso-639-1'; import 'linkify-plugin-hashtag'; @@ -6,22 +7,22 @@ import { nip19 } from 'nostr-tools'; import { z } from 'zod'; import { type AppController } from '@/app.ts'; -import { accountFromPubkey, renderAccount } from '@/views/mastodon/accounts.ts'; -import { addTag, deleteTag } from '@/utils/tags.ts'; -import { asyncReplaceAll } from '@/utils/text.ts'; import { Conf } from '@/config.ts'; import { DittoDB } from '@/db/DittoDB.ts'; +import { DittoUpload, dittoUploads } from '@/DittoUploads.ts'; import { DittoEvent } from '@/interfaces/DittoEvent.ts'; import { getAncestors, getAuthor, getDescendants, getEvent } from '@/queries.ts'; -import { getUnattachedMediaByIds } from '@/db/unattached-media.ts'; +import { addTag, deleteTag } from '@/utils/tags.ts'; +import { asyncReplaceAll } from '@/utils/text.ts'; import { lookupPubkey } from '@/utils/lookup.ts'; -import { renderEventAccounts } from '@/views.ts'; -import { renderReblog, renderStatus } from '@/views/mastodon/statuses.ts'; import { Storages } from '@/storages.ts'; import { hydrateEvents, purifyEvent } from '@/storages/hydrate.ts'; import { createEvent, paginated, paginatedList, parseBody, updateListEvent } from '@/utils/api.ts'; import { getInvoice, getLnurl } from '@/utils/lnurl.ts'; import { getZapSplits } from '@/utils/zap-split.ts'; +import { renderEventAccounts } from '@/views.ts'; +import { accountFromPubkey, renderAccount } from '@/views/mastodon/accounts.ts'; +import { renderReblog, renderStatus } from '@/views/mastodon/statuses.ts'; const createStatusSchema = z.object({ in_reply_to_id: n.id().nullish(), @@ -63,7 +64,6 @@ const statusController: AppController = async (c) => { const createStatusController: AppController = async (c) => { const body = await parseBody(c.req.raw); const result = createStatusSchema.safeParse(body); - const { kysely } = await DittoDB.getInstance(); const store = c.get('store'); if (!result.success) { @@ -112,10 +112,18 @@ const createStatusController: AppController = async (c) => { tags.push(['l', data.language, 'ISO-639-1']); } - const media = data.media_ids?.length ? await getUnattachedMediaByIds(kysely, data.media_ids) : []; + const media: DittoUpload[] = (data.media_ids ?? []).map((id) => { + const upload = dittoUploads.get(id); - const imeta: string[][] = media.map(({ data }) => { - const values: string[] = data.map((tag) => tag.join(' ')); + if (!upload) { + throw new HTTPException(422, { message: 'Uploaded attachment is no longer available.' }); + } + + return upload; + }); + + const imeta: string[][] = media.map(({ tags }) => { + const values: string[] = tags.map((tag) => tag.join(' ')); return ['imeta', ...values]; }); @@ -165,7 +173,7 @@ const createStatusController: AppController = async (c) => { } const mediaUrls: string[] = media - .map(({ data }) => data.find(([name]) => name === 'url')?.[1]) + .map(({ url }) => url) .filter((url): url is string => Boolean(url)); const quoteCompat = data.quote_id ? `\n\nnostr:${nip19.noteEncode(data.quote_id)}` : ''; diff --git a/src/db/DittoTables.ts b/src/db/DittoTables.ts index 69356649..09bf3e43 100644 --- a/src/db/DittoTables.ts +++ b/src/db/DittoTables.ts @@ -1,6 +1,5 @@ export interface DittoTables { nip46_tokens: NIP46TokenRow; - unattached_media: UnattachedMediaRow; author_stats: AuthorStatsRow; event_stats: EventStatsRow; pubkey_domains: PubkeyDomainRow; @@ -33,14 +32,6 @@ interface NIP46TokenRow { connected_at: Date; } -interface UnattachedMediaRow { - id: string; - pubkey: string; - url: string; - data: string; - uploaded_at: number; -} - interface PubkeyDomainRow { pubkey: string; domain: string; diff --git a/src/db/migrations/031_rm_unattached_media.ts b/src/db/migrations/031_rm_unattached_media.ts new file mode 100644 index 00000000..febd85e1 --- /dev/null +++ b/src/db/migrations/031_rm_unattached_media.ts @@ -0,0 +1,34 @@ +import { Kysely } from 'kysely'; + +export async function up(db: Kysely): Promise { + await db.schema.dropTable('unattached_media').execute(); +} + +export async function down(db: Kysely): Promise { + await db.schema + .createTable('unattached_media') + .addColumn('id', 'text', (c) => c.primaryKey()) + .addColumn('pubkey', 'text', (c) => c.notNull()) + .addColumn('url', 'text', (c) => c.notNull()) + .addColumn('data', 'text', (c) => c.notNull()) + .addColumn('uploaded_at', 'bigint', (c) => c.notNull()) + .execute(); + + await db.schema + .createIndex('unattached_media_id') + .on('unattached_media') + .column('id') + .execute(); + + await db.schema + .createIndex('unattached_media_pubkey') + .on('unattached_media') + .column('pubkey') + .execute(); + + await db.schema + .createIndex('unattached_media_url') + .on('unattached_media') + .column('url') + .execute(); +} diff --git a/src/db/unattached-media.ts b/src/db/unattached-media.ts deleted file mode 100644 index 260a9d5a..00000000 --- a/src/db/unattached-media.ts +++ /dev/null @@ -1,88 +0,0 @@ -import { Kysely } from 'kysely'; - -import { DittoDB } from '@/db/DittoDB.ts'; -import { DittoTables } from '@/db/DittoTables.ts'; - -interface UnattachedMedia { - id: string; - pubkey: string; - url: string; - /** NIP-94 tags. */ - data: string[][]; - uploaded_at: number; -} - -/** Add unattached media into the database. */ -async function insertUnattachedMedia(media: UnattachedMedia) { - const { kysely } = await DittoDB.getInstance(); - await kysely.insertInto('unattached_media') - .values({ ...media, data: JSON.stringify(media.data) }) - .execute(); - - return media; -} - -/** Select query for unattached media. */ -function selectUnattachedMediaQuery(kysely: Kysely) { - return kysely.selectFrom('unattached_media') - .select([ - 'unattached_media.id', - 'unattached_media.pubkey', - 'unattached_media.url', - 'unattached_media.data', - 'unattached_media.uploaded_at', - ]); -} - -/** Delete unattached media by URL. */ -async function deleteUnattachedMediaByUrl(url: string) { - const { kysely } = await DittoDB.getInstance(); - return kysely.deleteFrom('unattached_media') - .where('url', '=', url) - .execute(); -} - -/** Get unattached media by IDs. */ -async function getUnattachedMediaByIds(kysely: Kysely, ids: string[]): Promise { - if (!ids.length) return []; - - const results = await selectUnattachedMediaQuery(kysely) - .where('id', 'in', ids) - .execute(); - - return results.map((row) => ({ - ...row, - data: JSON.parse(row.data), - })); -} - -async function setMediaDescription(id: string, desc = '') { - const { kysely } = await DittoDB.getInstance(); - const existing = await selectUnattachedMediaQuery(kysely).where('id', '=', id).executeTakeFirst(); - if (!existing) return false; - const parsed = (await JSON.parse(existing.data) as string[][]).filter((itm) => itm[0] !== 'alt'); - parsed.push(['alt', desc]); - await kysely.updateTable('unattached_media') - .set({ data: JSON.stringify(parsed) }) - .execute(); - return true; -} - -/** Delete rows as an event with media is being created. */ -async function deleteAttachedMedia(pubkey: string, urls: string[]): Promise { - if (!urls.length) return; - const { kysely } = await DittoDB.getInstance(); - await kysely.deleteFrom('unattached_media') - .where('pubkey', '=', pubkey) - .where('url', 'in', urls) - .execute(); -} - -export { - deleteAttachedMedia, - deleteUnattachedMediaByUrl, - getUnattachedMediaByIds, - insertUnattachedMedia, - setMediaDescription, - type UnattachedMedia, -}; diff --git a/src/pipeline.ts b/src/pipeline.ts index 15c1ef34..dd59cb8d 100644 --- a/src/pipeline.ts +++ b/src/pipeline.ts @@ -6,7 +6,6 @@ import { z } from 'zod'; import { Conf } from '@/config.ts'; import { DittoDB } from '@/db/DittoDB.ts'; -import { deleteAttachedMedia } from '@/db/unattached-media.ts'; import { DittoEvent } from '@/interfaces/DittoEvent.ts'; import { pipelineEventsCounter, policyEventsCounter } from '@/metrics.ts'; import { RelayError } from '@/RelayError.ts'; @@ -61,7 +60,6 @@ async function handleEvent(event: DittoEvent, signal: AbortSignal): Promise { +): Promise { const uploader = c.get('uploader'); if (!uploader) { throw new HTTPException(500, { @@ -36,11 +36,15 @@ export async function uploadFile( tags.push(['alt', description]); } - return insertUnattachedMedia({ + const upload = { id: crypto.randomUUID(), - pubkey, url, - data: tags, - uploaded_at: Date.now(), - }); + tags, + pubkey, + uploadedAt: new Date(), + }; + + dittoUploads.set(upload.id, upload); + + return upload; } diff --git a/src/views/mastodon/attachments.ts b/src/views/mastodon/attachments.ts index 9320f604..4e9401fd 100644 --- a/src/views/mastodon/attachments.ts +++ b/src/views/mastodon/attachments.ts @@ -3,9 +3,9 @@ import { getUrlMediaType } from '@/utils/media.ts'; /** Render Mastodon media attachment. */ function renderAttachment( - media: { id?: string; data: string[][] }, + media: { id?: string; tags: string[][] }, ): (MastodonAttachment & { cid?: string }) | undefined { - const { id, data: tags } = media; + const { id, tags } = media; const url = tags.find(([name]) => name === 'url')?.[1]; diff --git a/src/views/mastodon/statuses.ts b/src/views/mastodon/statuses.ts index bc42765b..e21c9e1c 100644 --- a/src/views/mastodon/statuses.ts +++ b/src/views/mastodon/statuses.ts @@ -125,9 +125,9 @@ async function renderStatus(event: DittoEvent, opts: RenderStatusOpts): Promise< pinned: Boolean(pinEvent), reblog: null, application: null, - media_attachments: media.map((m) => renderAttachment({ data: m })).filter((m): m is MastodonAttachment => - Boolean(m) - ), + media_attachments: media + .map((m) => renderAttachment({ tags: m })) + .filter((m): m is MastodonAttachment => Boolean(m)), mentions, tags: [], emojis: renderEmojis(event), From 99a25e1e18c502d7d6abb227f99333a87f99a991 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 7 Sep 2024 10:32:14 -0500 Subject: [PATCH 21/82] media: fix setting description --- src/DittoUploads.ts | 1 - src/controllers/api/media.ts | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/src/DittoUploads.ts b/src/DittoUploads.ts index 044d3585..0024b1bf 100644 --- a/src/DittoUploads.ts +++ b/src/DittoUploads.ts @@ -5,7 +5,6 @@ import { Time } from '@/utils/time.ts'; export interface DittoUpload { id: string; pubkey: string; - description?: string; url: string; tags: string[][]; uploadedAt: Date; diff --git a/src/controllers/api/media.ts b/src/controllers/api/media.ts index 9dc2de27..7dc398ca 100644 --- a/src/controllers/api/media.ts +++ b/src/controllers/api/media.ts @@ -54,7 +54,7 @@ const updateMediaController: AppController = async (c) => { dittoUploads.set(id, { ...upload, - description, + tags: upload.tags.filter(([name]) => name !== 'alt').concat([['alt', description]]), }); return c.json({ message: 'ok' }, 200); From ac4a63bdcc7799f72297eb94e2364875ce40baa2 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 7 Sep 2024 10:43:01 -0500 Subject: [PATCH 22/82] deno task soapbox: don't prompt to overwrite --- deno.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deno.json b/deno.json index bd78ae23..64331504 100644 --- a/deno.json +++ b/deno.json @@ -17,7 +17,7 @@ "admin:role": "deno run -A scripts/admin-role.ts", "setup": "deno run -A scripts/setup.ts", "stats:recompute": "deno run -A scripts/stats-recompute.ts", - "soapbox": "curl -O https://dl.soapbox.pub/main/soapbox.zip && mkdir -p public && mv soapbox.zip public/ && cd public/ && unzip soapbox.zip && rm soapbox.zip", + "soapbox": "curl -O https://dl.soapbox.pub/main/soapbox.zip && mkdir -p public && mv soapbox.zip public/ && cd public/ && unzip -o soapbox.zip && rm soapbox.zip", "trends": "deno run -A scripts/trends.ts", "clean:deps": "deno cache --reload src/app.ts" }, From 23f603842731ebb40ac84d22ece70c5dfdf90978 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 7 Sep 2024 10:54:48 -0500 Subject: [PATCH 23/82] Add ansible playbook to update multiple Ditto servers at once --- ansible/playbook.yml | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 ansible/playbook.yml diff --git a/ansible/playbook.yml b/ansible/playbook.yml new file mode 100644 index 00000000..a2d8226e --- /dev/null +++ b/ansible/playbook.yml @@ -0,0 +1,23 @@ +--- +- name: Update Ditto + hosts: all + become: true + tasks: + - name: Update Soapbox + shell: + cmd: deno task soapbox + chdir: /opt/ditto + become_user: ditto + + - name: Update ditto from the main branch + git: + repo: 'https://gitlab.com/soapbox-pub/ditto.git' + dest: '/opt/ditto' + version: main + become_user: ditto + + - name: Restart ditto service + systemd: + name: ditto + state: restarted + become_user: root From a19b7fbe9efe56b8d9f45bba371c7383b751987f Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 7 Sep 2024 13:07:47 -0500 Subject: [PATCH 24/82] Fetch link previews with facebookexternalhit instead of WhatsApp/2 by default, make it configurable --- src/config.ts | 4 ++++ src/utils/unfurl.ts | 6 +++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/src/config.ts b/src/config.ts index 0cb71509..7954e744 100644 --- a/src/config.ts +++ b/src/config.ts @@ -258,6 +258,10 @@ class Conf { 'i', ); } + /** User-Agent to use when fetching link previews. Pretend to be Facebook by default. */ + static get fetchUserAgent(): string { + return Deno.env.get('DITTO_FETCH_USER_AGENT') ?? 'facebookexternalhit'; + } /** Path to the custom policy module. Must be an absolute path, https:, npm:, or jsr: URI. */ static get policy(): string { return Deno.env.get('DITTO_POLICY') || new URL('../data/policy.ts', import.meta.url).pathname; diff --git a/src/utils/unfurl.ts b/src/utils/unfurl.ts index a0ab1d7b..8123c423 100644 --- a/src/utils/unfurl.ts +++ b/src/utils/unfurl.ts @@ -3,6 +3,7 @@ import Debug from '@soapbox/stickynotes/debug'; import DOMPurify from 'isomorphic-dompurify'; import { unfurl } from 'unfurl.js'; +import { Conf } from '@/config.ts'; import { PreviewCard } from '@/entities/PreviewCard.ts'; import { Time } from '@/utils/time.ts'; import { fetchWorker } from '@/workers/fetch.ts'; @@ -15,7 +16,10 @@ async function unfurlCard(url: string, signal: AbortSignal): Promise fetchWorker(url, { - headers: { 'User-Agent': 'WhatsApp/2' }, + headers: { + 'Accept': 'text/html, application/xhtml+xml', + 'User-Agent': Conf.fetchUserAgent, + }, signal, }), }); From 69329674e1ea3d50e29d4c65bed9e749c4a376af Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 7 Sep 2024 16:38:28 -0500 Subject: [PATCH 25/82] Add support for pglite --- .tool-versions | 2 +- deno.json | 2 ++ deno.lock | 14 +++++++++ src/config.ts | 3 +- src/db/DittoDB.ts | 12 ++++++-- src/db/adapters/DittoPglite.ts | 56 ++++++++++++++++++++++++++++++++++ 6 files changed, 84 insertions(+), 5 deletions(-) create mode 100644 src/db/adapters/DittoPglite.ts diff --git a/.tool-versions b/.tool-versions index 73d0a9db..900b9e20 100644 --- a/.tool-versions +++ b/.tool-versions @@ -1 +1 @@ -deno 1.45.5 \ No newline at end of file +deno 1.46.3 \ No newline at end of file diff --git a/deno.json b/deno.json index 64331504..d5db42c8 100644 --- a/deno.json +++ b/deno.json @@ -28,6 +28,7 @@ "@b-fuze/deno-dom": "jsr:@b-fuze/deno-dom@^0.1.47", "@bradenmacdonald/s3-lite-client": "jsr:@bradenmacdonald/s3-lite-client@^0.7.4", "@db/sqlite": "jsr:@db/sqlite@^0.11.1", + "@electric-sql/pglite": "npm:@electric-sql/pglite@^0.2.5", "@hono/hono": "jsr:@hono/hono@^4.4.6", "@isaacs/ttlcache": "npm:@isaacs/ttlcache@^1.4.1", "@lambdalisue/async": "jsr:@lambdalisue/async@^2.1.1", @@ -37,6 +38,7 @@ "@scure/base": "npm:@scure/base@^1.1.6", "@sentry/deno": "https://deno.land/x/sentry@7.112.2/index.mjs", "@soapbox/kysely-deno-sqlite": "jsr:@soapbox/kysely-deno-sqlite@^2.1.0", + "@soapbox/kysely-pglite": "jsr:@soapbox/kysely-pglite@^0.0.5", "@soapbox/stickynotes": "jsr:@soapbox/stickynotes@^0.4.0", "@std/assert": "jsr:@std/assert@^0.225.1", "@std/cli": "jsr:@std/cli@^0.223.0", diff --git a/deno.lock b/deno.lock index cb040522..e59e513c 100644 --- a/deno.lock +++ b/deno.lock @@ -22,6 +22,7 @@ "jsr:@nostrify/nostrify@^0.30.1": "jsr:@nostrify/nostrify@0.30.1", "jsr:@nostrify/types@^0.30.0": "jsr:@nostrify/types@0.30.0", "jsr:@soapbox/kysely-deno-sqlite@^2.1.0": "jsr:@soapbox/kysely-deno-sqlite@2.2.0", + "jsr:@soapbox/kysely-pglite@^0.0.5": "jsr:@soapbox/kysely-pglite@0.0.5", "jsr:@soapbox/stickynotes@^0.4.0": "jsr:@soapbox/stickynotes@0.4.0", "jsr:@std/assert@^0.213.1": "jsr:@std/assert@0.213.1", "jsr:@std/assert@^0.217.0": "jsr:@std/assert@0.217.0", @@ -53,6 +54,7 @@ "jsr:@std/path@^0.213.1": "jsr:@std/path@0.213.1", "jsr:@std/path@^0.221.0": "jsr:@std/path@0.221.0", "jsr:@std/streams@^0.223.0": "jsr:@std/streams@0.223.0", + "npm:@electric-sql/pglite@^0.2.5": "npm:@electric-sql/pglite@0.2.5", "npm:@isaacs/ttlcache@^1.4.1": "npm:@isaacs/ttlcache@1.4.1", "npm:@noble/hashes@^1.4.0": "npm:@noble/hashes@1.4.0", "npm:@scure/base@^1.1.6": "npm:@scure/base@1.1.6", @@ -257,6 +259,12 @@ "npm:kysely@^0.27.2" ] }, + "@soapbox/kysely-pglite@0.0.5": { + "integrity": "87fc586d46cffede8dcc18598f41411db296130c14f22b044d6a5538fd6e59b5", + "dependencies": [ + "npm:kysely@^0.27.4" + ] + }, "@soapbox/stickynotes@0.4.0": { "integrity": "60bfe61ab3d7e04bf708273b1e2d391a59534bdf29e54160e98d7afd328ca1ec" }, @@ -397,6 +405,10 @@ } }, "npm": { + "@electric-sql/pglite@0.2.5": { + "integrity": "sha512-LrMX2kX0mCVN4xkhIDv1KBVukWtoOI/+P9MDQgHX5QEeZCi5S60LZOa0VWXjufPEz7mJtbuXWJRujD++t0gsHA==", + "dependencies": {} + }, "@isaacs/ttlcache@1.4.1": { "integrity": "sha512-RQgQ4uQ+pLbqXfOmieB91ejmLwvSgv9nLx6sT6sD83s7umBypgg+OIBOBbEUiJXrfpnp9j0mRhYYdzp9uqq3lA==", "dependencies": {} @@ -1920,6 +1932,7 @@ "jsr:@nostrify/db@^0.31.2", "jsr:@nostrify/nostrify@^0.30.1", "jsr:@soapbox/kysely-deno-sqlite@^2.1.0", + "jsr:@soapbox/kysely-pglite@^0.0.5", "jsr:@soapbox/stickynotes@^0.4.0", "jsr:@std/assert@^0.225.1", "jsr:@std/cli@^0.223.0", @@ -1930,6 +1943,7 @@ "jsr:@std/json@^0.223.0", "jsr:@std/media-types@^0.224.1", "jsr:@std/streams@^0.223.0", + "npm:@electric-sql/pglite@^0.2.5", "npm:@isaacs/ttlcache@^1.4.1", "npm:@noble/secp256k1@^2.0.0", "npm:@scure/base@^1.1.6", diff --git a/src/config.ts b/src/config.ts index 7954e744..145ed9e4 100644 --- a/src/config.ts +++ b/src/config.ts @@ -82,7 +82,7 @@ class Conf { * ``` */ static get databaseUrl(): string { - return Deno.env.get('DATABASE_URL') ?? 'sqlite://data/db.sqlite3'; + return Deno.env.get('DATABASE_URL') ?? 'pglite://data/pgdata'; } static db = { get url(): url.UrlWithStringQuery { @@ -92,6 +92,7 @@ class Conf { switch (Conf.db.url.protocol) { case 'sqlite:': return 'sqlite'; + case 'pglite:': case 'postgres:': case 'postgresql:': return 'postgres'; diff --git a/src/db/DittoDB.ts b/src/db/DittoDB.ts index 5ed5d15e..40d03e5f 100644 --- a/src/db/DittoDB.ts +++ b/src/db/DittoDB.ts @@ -5,6 +5,7 @@ import { NDatabaseSchema, NPostgresSchema } from '@nostrify/db'; import { FileMigrationProvider, Kysely, Migrator } from 'kysely'; import { Conf } from '@/config.ts'; +import { DittoPglite } from '@/db/adapters/DittoPglite.ts'; import { DittoPostgres } from '@/db/adapters/DittoPostgres.ts'; import { DittoSQLite } from '@/db/adapters/DittoSQLite.ts'; import { DittoTables } from '@/db/DittoTables.ts'; @@ -30,12 +31,17 @@ export class DittoDB { static async _getInstance(): Promise { const result = {} as DittoDatabase; - switch (Conf.db.dialect) { - case 'sqlite': + switch (Conf.db.url.protocol) { + case 'sqlite:': result.dialect = 'sqlite'; result.kysely = await DittoSQLite.getInstance(); break; - case 'postgres': + case 'pglite:': + result.dialect = 'postgres'; + result.kysely = await DittoPglite.getInstance(); + break; + case 'postgres:': + case 'postgresql:': result.dialect = 'postgres'; result.kysely = await DittoPostgres.getInstance(); break; diff --git a/src/db/adapters/DittoPglite.ts b/src/db/adapters/DittoPglite.ts new file mode 100644 index 00000000..ef035a3b --- /dev/null +++ b/src/db/adapters/DittoPglite.ts @@ -0,0 +1,56 @@ +import { PGlite } from '@electric-sql/pglite'; +import { NPostgresSchema } from '@nostrify/db'; +import { PgliteDialect } from '@soapbox/kysely-pglite'; +import { Kysely } from 'kysely'; + +import { Conf } from '@/config.ts'; +import { DittoTables } from '@/db/DittoTables.ts'; +import { KyselyLogger } from '@/db/KyselyLogger.ts'; + +export class DittoPglite { + static db: Kysely & Kysely | undefined; + + // deno-lint-ignore require-await + static async getInstance(): Promise & Kysely> { + if (!this.db) { + this.db = new Kysely({ + dialect: new PgliteDialect({ + database: new PGlite(this.path), + }), + log: KyselyLogger, + }) as Kysely & Kysely; + } + + return this.db; + } + + static get poolSize() { + return 1; + } + + static get availableConnections(): number { + return 1; + } + + /** Get the relative or absolute path based on the `DATABASE_URL`. */ + static get path(): string | undefined { + if (Conf.databaseUrl === 'pglite://:memory:') { + return undefined; + } + + const { host, pathname } = Conf.db.url; + + if (!pathname) return ''; + + // Get relative path. + if (host === '') { + return pathname; + } else if (host === '.') { + return pathname; + } else if (host) { + return host + pathname; + } + + return ''; + } +} From 819957da49c1e0fb7e924872cff5ea7aa06fc2e5 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 7 Sep 2024 20:56:50 -0500 Subject: [PATCH 26/82] Upgrade kysely-pglite --- deno.json | 2 +- deno.lock | 31 ++++++++++++++++++++++++------- 2 files changed, 25 insertions(+), 8 deletions(-) diff --git a/deno.json b/deno.json index d5db42c8..6cd17018 100644 --- a/deno.json +++ b/deno.json @@ -38,7 +38,7 @@ "@scure/base": "npm:@scure/base@^1.1.6", "@sentry/deno": "https://deno.land/x/sentry@7.112.2/index.mjs", "@soapbox/kysely-deno-sqlite": "jsr:@soapbox/kysely-deno-sqlite@^2.1.0", - "@soapbox/kysely-pglite": "jsr:@soapbox/kysely-pglite@^0.0.5", + "@soapbox/kysely-pglite": "jsr:@soapbox/kysely-pglite@^0.0.6", "@soapbox/stickynotes": "jsr:@soapbox/stickynotes@^0.4.0", "@std/assert": "jsr:@std/assert@^0.225.1", "@std/cli": "jsr:@std/cli@^0.223.0", diff --git a/deno.lock b/deno.lock index e59e513c..bbaf3011 100644 --- a/deno.lock +++ b/deno.lock @@ -2,7 +2,7 @@ "version": "3", "packages": { "specifiers": { - "jsr:@b-fuze/deno-dom@^0.1.47": "jsr:@b-fuze/deno-dom@0.1.47", + "jsr:@b-fuze/deno-dom@^0.1.47": "jsr:@b-fuze/deno-dom@0.1.48", "jsr:@bradenmacdonald/s3-lite-client@^0.7.4": "jsr:@bradenmacdonald/s3-lite-client@0.7.6", "jsr:@db/sqlite@^0.11.1": "jsr:@db/sqlite@0.11.1", "jsr:@denosaurs/plug@1": "jsr:@denosaurs/plug@1.0.6", @@ -12,7 +12,7 @@ "jsr:@gleasonator/policy@0.4.0": "jsr:@gleasonator/policy@0.4.0", "jsr:@gleasonator/policy@0.4.1": "jsr:@gleasonator/policy@0.4.1", "jsr:@gleasonator/policy@0.4.2": "jsr:@gleasonator/policy@0.4.2", - "jsr:@hono/hono@^4.4.6": "jsr:@hono/hono@4.5.9", + "jsr:@hono/hono@^4.4.6": "jsr:@hono/hono@4.5.11", "jsr:@lambdalisue/async@^2.1.1": "jsr:@lambdalisue/async@2.1.1", "jsr:@nostrify/db@^0.31.2": "jsr:@nostrify/db@0.31.2", "jsr:@nostrify/nostrify@^0.22.1": "jsr:@nostrify/nostrify@0.22.5", @@ -22,7 +22,7 @@ "jsr:@nostrify/nostrify@^0.30.1": "jsr:@nostrify/nostrify@0.30.1", "jsr:@nostrify/types@^0.30.0": "jsr:@nostrify/types@0.30.0", "jsr:@soapbox/kysely-deno-sqlite@^2.1.0": "jsr:@soapbox/kysely-deno-sqlite@2.2.0", - "jsr:@soapbox/kysely-pglite@^0.0.5": "jsr:@soapbox/kysely-pglite@0.0.5", + "jsr:@soapbox/kysely-pglite@^0.0.6": "jsr:@soapbox/kysely-pglite@0.0.6", "jsr:@soapbox/stickynotes@^0.4.0": "jsr:@soapbox/stickynotes@0.4.0", "jsr:@std/assert@^0.213.1": "jsr:@std/assert@0.213.1", "jsr:@std/assert@^0.217.0": "jsr:@std/assert@0.217.0", @@ -40,13 +40,14 @@ "jsr:@std/encoding@^0.221.0": "jsr:@std/encoding@0.221.0", "jsr:@std/encoding@^0.224.0": "jsr:@std/encoding@0.224.3", "jsr:@std/encoding@^0.224.1": "jsr:@std/encoding@0.224.3", + "jsr:@std/encoding@^1.0.4": "jsr:@std/encoding@1.0.4", "jsr:@std/fmt@0.213.1": "jsr:@std/fmt@0.213.1", "jsr:@std/fmt@^0.221.0": "jsr:@std/fmt@0.221.0", "jsr:@std/fs@0.213.1": "jsr:@std/fs@0.213.1", "jsr:@std/fs@^0.221.0": "jsr:@std/fs@0.221.0", "jsr:@std/fs@^0.229.3": "jsr:@std/fs@0.229.3", "jsr:@std/internal@^1.0.0": "jsr:@std/internal@1.0.1", - "jsr:@std/io@^0.224": "jsr:@std/io@0.224.6", + "jsr:@std/io@^0.224": "jsr:@std/io@0.224.7", "jsr:@std/json@^0.223.0": "jsr:@std/json@0.223.0", "jsr:@std/media-types@^0.224.1": "jsr:@std/media-types@0.224.1", "jsr:@std/path@0.213.1": "jsr:@std/path@0.213.1", @@ -103,6 +104,9 @@ "jsr:@denosaurs/plug@1.0.3" ] }, + "@b-fuze/deno-dom@0.1.48": { + "integrity": "bf5b591aef2e9e9c59adfcbb93a9ecd45bab5b7c8263625beafa5c8f1662e7da" + }, "@bradenmacdonald/s3-lite-client@0.7.6": { "integrity": "2b5976dca95d207dc88e23f9807e3eecbc441b0cf547dcda5784afe6668404d1", "dependencies": [ @@ -167,6 +171,9 @@ "@hono/hono@4.5.1": { "integrity": "459748ed4d4146c6e4bdff0213ff1ac44749904066ae02e7550d6c7f28c9bc4c" }, + "@hono/hono@4.5.11": { + "integrity": "5bd6b1a3a503efb746fdcf0aae3ac536dd09229d372988bde5db0798ef64ae4f" + }, "@hono/hono@4.5.3": { "integrity": "429923b2b3c6586a1450862328d61a1346fee5841e8ae86c494250475057213c" }, @@ -259,9 +266,10 @@ "npm:kysely@^0.27.2" ] }, - "@soapbox/kysely-pglite@0.0.5": { - "integrity": "87fc586d46cffede8dcc18598f41411db296130c14f22b044d6a5538fd6e59b5", + "@soapbox/kysely-pglite@0.0.6": { + "integrity": "df88a2610c0bbfe382a0d4d060a1968458b96b79e990e4d7b85744ca819f00d9", "dependencies": [ + "jsr:@std/encoding@^1.0.4", "npm:kysely@^0.27.4" ] }, @@ -317,6 +325,9 @@ "@std/encoding@0.224.3": { "integrity": "5e861b6d81be5359fad4155e591acf17c0207b595112d1840998bb9f476dbdaf" }, + "@std/encoding@1.0.4": { + "integrity": "2266cd516b32369e3dc5695717c96bf88343a1f761d6e6187a02a2bbe2af86ae" + }, "@std/fmt@0.213.1": { "integrity": "a06d31777566d874b9c856c10244ac3e6b660bdec4c82506cd46be052a1082c3" }, @@ -376,6 +387,12 @@ "jsr:@std/bytes@^1.0.2" ] }, + "@std/io@0.224.7": { + "integrity": "a70848793c44a7c100926571a8c9be68ba85487bfcd4d0540d86deabe1123dc9", + "dependencies": [ + "jsr:@std/bytes@^1.0.2" + ] + }, "@std/json@0.223.0": { "integrity": "9a4a255931dd0397924c6b10bb6a72fe3e28ddd876b981ada2e3b8dd0764163f" }, @@ -1932,7 +1949,7 @@ "jsr:@nostrify/db@^0.31.2", "jsr:@nostrify/nostrify@^0.30.1", "jsr:@soapbox/kysely-deno-sqlite@^2.1.0", - "jsr:@soapbox/kysely-pglite@^0.0.5", + "jsr:@soapbox/kysely-pglite@^0.0.6", "jsr:@soapbox/stickynotes@^0.4.0", "jsr:@std/assert@^0.225.1", "jsr:@std/cli@^0.223.0", From 8a94e8deb3f06f5ac9deffffc9b270f2d74d9ea8 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 8 Sep 2024 05:38:33 -0500 Subject: [PATCH 27/82] grafana: fix db effort query --- grafana/Ditto-Dashboard.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/grafana/Ditto-Dashboard.json b/grafana/Ditto-Dashboard.json index 222b727c..90905f2d 100644 --- a/grafana/Ditto-Dashboard.json +++ b/grafana/Ditto-Dashboard.json @@ -457,7 +457,7 @@ "uid": "${prometheus}" }, "editorMode": "code", - "expr": "rate(ditto_db_query_duration_ms_sum[$__rate_interval]) / rate(db_query_total[$__rate_interval])", + "expr": "rate(ditto_db_query_duration_ms_sum[$__rate_interval]) / rate(ditto_db_queries_total[$__rate_interval])", "instant": false, "interval": "", "legendFormat": "Effort", @@ -2112,6 +2112,6 @@ "timezone": "browser", "title": "Ditto", "uid": "ddps3ap51fv28d", - "version": 59, + "version": 60, "weekStart": "" } \ No newline at end of file From 2309fd72d1d167fb9d6078efe50c53a59c456ae4 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 8 Sep 2024 09:05:23 -0500 Subject: [PATCH 28/82] Don't limit statuses to kind 1 --- src/controllers/api/statuses.ts | 1 - src/queries.ts | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/src/controllers/api/statuses.ts b/src/controllers/api/statuses.ts index 47adc572..05b5022c 100644 --- a/src/controllers/api/statuses.ts +++ b/src/controllers/api/statuses.ts @@ -50,7 +50,6 @@ const statusController: AppController = async (c) => { const id = c.req.param('id'); const event = await getEvent(id, { - kind: 1, signal: AbortSignal.timeout(1500), }); diff --git a/src/queries.ts b/src/queries.ts index 9bce58ca..9ee86a36 100644 --- a/src/queries.ts +++ b/src/queries.ts @@ -74,7 +74,7 @@ async function getAncestors(store: NStore, event: NostrEvent, result: NostrEvent const inReplyTo = replyTag ? replyTag[1] : undefined; if (inReplyTo) { - const [parentEvent] = await store.query([{ kinds: [1], ids: [inReplyTo], until: event.created_at, limit: 1 }]); + const [parentEvent] = await store.query([{ ids: [inReplyTo], until: event.created_at, limit: 1 }]); if (parentEvent) { result.push(parentEvent); From 9331ab4ac123a352703385732afc3389025bdd8a Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 8 Sep 2024 13:31:09 -0500 Subject: [PATCH 29/82] grafana: add system usage, break up into rows --- grafana/Ditto-Dashboard.json | 391 ++++++++++++++++++----------------- 1 file changed, 205 insertions(+), 186 deletions(-) diff --git a/grafana/Ditto-Dashboard.json b/grafana/Ditto-Dashboard.json index 90905f2d..0c161eb4 100644 --- a/grafana/Ditto-Dashboard.json +++ b/grafana/Ditto-Dashboard.json @@ -19,6 +19,12 @@ ], "__elements": {}, "__requires": [ + { + "type": "panel", + "id": "bargauge", + "name": "Bar gauge", + "version": "" + }, { "type": "panel", "id": "gauge", @@ -29,7 +35,7 @@ "type": "grafana", "id": "grafana", "name": "Grafana", - "version": "11.2.0" + "version": "11.1.4" }, { "type": "datasource", @@ -85,6 +91,19 @@ "id": null, "links": [], "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 22, + "panels": [], + "title": "Overview", + "type": "row" + }, { "datasource": { "default": false, @@ -104,7 +123,6 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, - "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 25, "gradientMode": "none", @@ -212,7 +230,7 @@ "h": 10, "w": 24, "x": 0, - "y": 0 + "y": 1 }, "id": 10, "options": { @@ -327,7 +345,7 @@ "h": 8, "w": 6, "x": 0, - "y": 10 + "y": 11 }, "id": 8, "options": { @@ -345,7 +363,7 @@ "showThresholdMarkers": true, "sizing": "auto" }, - "pluginVersion": "11.2.0", + "pluginVersion": "11.1.4", "targets": [ { "datasource": { @@ -368,52 +386,15 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "description": "How hard the database is working on a per-query basis. It's query time divided by number of queries. Lower is better.", + "description": "Usage of system resources.", "fieldConfig": { "defaults": { "color": { - "fixedColor": "red", - "mode": "continuous-GrYlRd", - "seriesBy": "last" + "mode": "continuous-GrYlRd" }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "scheme", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineStyle": { - "fill": "solid" - }, - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "fieldMinMax": false, "mappings": [], + "max": 100, + "min": 0, "thresholds": { "mode": "absolute", "steps": [ @@ -426,47 +407,66 @@ "value": 80 } ] - } + }, + "unit": "percent" }, "overrides": [] }, "gridPos": { - "h": 8, + "h": 5, "w": 12, "x": 6, - "y": 10 + "y": 11 }, - "id": 11, + "id": 20, "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false + "displayMode": "lcd", + "maxVizHeight": 300, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false }, - "tooltip": { - "maxHeight": 600, - "mode": "single", - "sort": "none" - } + "showUnfilled": true, + "sizing": "auto", + "valueMode": "color" }, + "pluginVersion": "11.1.4", "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "100 * (1 - avg(rate(node_cpu_seconds_total{mode=\"idle\"}[$__rate_interval])))", + "instant": false, + "legendFormat": "CPU", + "range": true, + "refId": "A" + }, { "datasource": { "type": "prometheus", "uid": "${prometheus}" }, "editorMode": "code", - "expr": "rate(ditto_db_query_duration_ms_sum[$__rate_interval]) / rate(ditto_db_queries_total[$__rate_interval])", + "expr": "(1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)) * 100", + "hide": false, "instant": false, - "interval": "", - "legendFormat": "Effort", + "legendFormat": "RAM", "range": true, - "refId": "A" + "refId": "B" } ], - "title": "Database Effort", - "type": "timeseries" + "title": "System Usage", + "type": "bargauge" }, { "datasource": { @@ -486,7 +486,6 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, - "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -533,7 +532,7 @@ "h": 8, "w": 6, "x": 18, - "y": 10 + "y": 11 }, "id": 6, "options": { @@ -566,6 +565,103 @@ "title": "Database Queries", "type": "timeseries" }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Total number of Nostr clients currently connected to the Nostr relay.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 12, + "x": 6, + "y": 16 + }, + "id": 2, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "ditto_relay_connections", + "instant": false, + "legendFormat": "Connections", + "range": true, + "refId": "A" + } + ], + "title": "Relay Connections", + "type": "timeseries" + }, { "datasource": { "type": "prometheus", @@ -584,7 +680,6 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, - "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", @@ -662,7 +757,7 @@ "h": 8, "w": 6, "x": 0, - "y": 18 + "y": 19 }, "id": 12, "options": { @@ -708,105 +803,6 @@ "title": "Policy Events", "type": "timeseries" }, - { - "datasource": { - "default": false, - "type": "prometheus", - "uid": "${prometheus}" - }, - "description": "Total number of Nostr clients currently connected to the Nostr relay.", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 6, - "y": 18 - }, - "id": 2, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "maxHeight": 600, - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "editorMode": "code", - "expr": "ditto_relay_connections", - "instant": false, - "legendFormat": "Connections", - "range": true, - "refId": "A" - } - ], - "title": "Relay Connections", - "type": "timeseries" - }, { "datasource": { "default": false, @@ -826,7 +822,6 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, - "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -874,7 +869,7 @@ "h": 8, "w": 6, "x": 18, - "y": 18 + "y": 19 }, "id": 7, "options": { @@ -907,6 +902,19 @@ "title": "Database Query Time", "type": "timeseries" }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 27 + }, + "id": 21, + "panels": [], + "title": "Database", + "type": "row" + }, { "datasource": { "default": false, @@ -1084,7 +1092,7 @@ "h": 11, "w": 15, "x": 0, - "y": 26 + "y": 28 }, "id": 13, "options": { @@ -1106,7 +1114,7 @@ } ] }, - "pluginVersion": "11.2.0", + "pluginVersion": "11.1.4", "targets": [ { "datasource": { @@ -1240,7 +1248,7 @@ "h": 11, "w": 9, "x": 15, - "y": 26 + "y": 28 }, "id": 14, "options": { @@ -1255,7 +1263,7 @@ }, "showHeader": true }, - "pluginVersion": "11.2.0", + "pluginVersion": "11.1.4", "targets": [ { "datasource": { @@ -1317,7 +1325,7 @@ "h": 9, "w": 7, "x": 0, - "y": 37 + "y": 39 }, "id": 16, "options": { @@ -1414,7 +1422,7 @@ "h": 9, "w": 8, "x": 7, - "y": 37 + "y": 39 }, "id": 17, "options": { @@ -1579,7 +1587,7 @@ "h": 9, "w": 9, "x": 15, - "y": 37 + "y": 39 }, "id": 18, "options": { @@ -1594,7 +1602,7 @@ }, "showHeader": true }, - "pluginVersion": "11.2.0", + "pluginVersion": "11.1.4", "targets": [ { "datasource": { @@ -1628,6 +1636,19 @@ "title": "Unused Indexes", "type": "table" }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 48 + }, + "id": 23, + "panels": [], + "title": "Nostr", + "type": "row" + }, { "datasource": { "default": false, @@ -1647,7 +1668,6 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, - "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -1758,7 +1778,7 @@ "h": 12, "w": 24, "x": 0, - "y": 46 + "y": 49 }, "id": 9, "options": { @@ -1812,7 +1832,6 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, - "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -1896,7 +1915,7 @@ "h": 12, "w": 24, "x": 0, - "y": 58 + "y": 61 }, "id": 5, "options": { @@ -1948,7 +1967,6 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, - "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -1995,7 +2013,7 @@ "h": 11, "w": 24, "x": 0, - "y": 70 + "y": 73 }, "id": 19, "options": { @@ -2081,6 +2099,7 @@ "name": "prometheus", "options": [], "query": "prometheus", + "queryValue": "", "refresh": 1, "regex": "", "skipUrlSync": false, @@ -2112,6 +2131,6 @@ "timezone": "browser", "title": "Ditto", "uid": "ddps3ap51fv28d", - "version": 60, + "version": 4, "weekStart": "" } \ No newline at end of file From 6a142721841e9299cf84550a4b622ac1907b03e2 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Tue, 10 Sep 2024 15:59:15 -0500 Subject: [PATCH 30/82] Make pglite work --- deno.json | 4 ++-- deno.lock | 61 ++++++++++++++++++++++++++++++++++++++++--------------- 2 files changed, 47 insertions(+), 18 deletions(-) diff --git a/deno.json b/deno.json index 6cd17018..840caf77 100644 --- a/deno.json +++ b/deno.json @@ -28,7 +28,7 @@ "@b-fuze/deno-dom": "jsr:@b-fuze/deno-dom@^0.1.47", "@bradenmacdonald/s3-lite-client": "jsr:@bradenmacdonald/s3-lite-client@^0.7.4", "@db/sqlite": "jsr:@db/sqlite@^0.11.1", - "@electric-sql/pglite": "npm:@electric-sql/pglite@^0.2.5", + "@electric-sql/pglite": "npm:@soapbox.pub/pglite@^0.2.10", "@hono/hono": "jsr:@hono/hono@^4.4.6", "@isaacs/ttlcache": "npm:@isaacs/ttlcache@^1.4.1", "@lambdalisue/async": "jsr:@lambdalisue/async@^2.1.1", @@ -38,7 +38,7 @@ "@scure/base": "npm:@scure/base@^1.1.6", "@sentry/deno": "https://deno.land/x/sentry@7.112.2/index.mjs", "@soapbox/kysely-deno-sqlite": "jsr:@soapbox/kysely-deno-sqlite@^2.1.0", - "@soapbox/kysely-pglite": "jsr:@soapbox/kysely-pglite@^0.0.6", + "@soapbox/kysely-pglite": "jsr:@soapbox/kysely-pglite@^0.0.1", "@soapbox/stickynotes": "jsr:@soapbox/stickynotes@^0.4.0", "@std/assert": "jsr:@std/assert@^0.225.1", "@std/cli": "jsr:@std/cli@^0.223.0", diff --git a/deno.lock b/deno.lock index bbaf3011..d9eee2d8 100644 --- a/deno.lock +++ b/deno.lock @@ -12,6 +12,7 @@ "jsr:@gleasonator/policy@0.4.0": "jsr:@gleasonator/policy@0.4.0", "jsr:@gleasonator/policy@0.4.1": "jsr:@gleasonator/policy@0.4.1", "jsr:@gleasonator/policy@0.4.2": "jsr:@gleasonator/policy@0.4.2", + "jsr:@gleasonator/policy@0.5.0": "jsr:@gleasonator/policy@0.5.0", "jsr:@hono/hono@^4.4.6": "jsr:@hono/hono@4.5.11", "jsr:@lambdalisue/async@^2.1.1": "jsr:@lambdalisue/async@2.1.1", "jsr:@nostrify/db@^0.31.2": "jsr:@nostrify/db@0.31.2", @@ -20,9 +21,12 @@ "jsr:@nostrify/nostrify@^0.22.5": "jsr:@nostrify/nostrify@0.22.5", "jsr:@nostrify/nostrify@^0.30.0": "jsr:@nostrify/nostrify@0.30.1", "jsr:@nostrify/nostrify@^0.30.1": "jsr:@nostrify/nostrify@0.30.1", - "jsr:@nostrify/types@^0.30.0": "jsr:@nostrify/types@0.30.0", + "jsr:@nostrify/nostrify@^0.31.0": "jsr:@nostrify/nostrify@0.31.0", + "jsr:@nostrify/policies@^0.33.0": "jsr:@nostrify/policies@0.33.0", + "jsr:@nostrify/types@^0.30.0": "jsr:@nostrify/types@0.30.1", + "jsr:@nostrify/types@^0.30.1": "jsr:@nostrify/types@0.30.1", "jsr:@soapbox/kysely-deno-sqlite@^2.1.0": "jsr:@soapbox/kysely-deno-sqlite@2.2.0", - "jsr:@soapbox/kysely-pglite@^0.0.6": "jsr:@soapbox/kysely-pglite@0.0.6", + "jsr:@soapbox/kysely-pglite@^0.0.1": "jsr:@soapbox/kysely-pglite@0.0.1", "jsr:@soapbox/stickynotes@^0.4.0": "jsr:@soapbox/stickynotes@0.4.0", "jsr:@std/assert@^0.213.1": "jsr:@std/assert@0.213.1", "jsr:@std/assert@^0.217.0": "jsr:@std/assert@0.217.0", @@ -40,7 +44,6 @@ "jsr:@std/encoding@^0.221.0": "jsr:@std/encoding@0.221.0", "jsr:@std/encoding@^0.224.0": "jsr:@std/encoding@0.224.3", "jsr:@std/encoding@^0.224.1": "jsr:@std/encoding@0.224.3", - "jsr:@std/encoding@^1.0.4": "jsr:@std/encoding@1.0.4", "jsr:@std/fmt@0.213.1": "jsr:@std/fmt@0.213.1", "jsr:@std/fmt@^0.221.0": "jsr:@std/fmt@0.221.0", "jsr:@std/fs@0.213.1": "jsr:@std/fs@0.213.1", @@ -55,12 +58,12 @@ "jsr:@std/path@^0.213.1": "jsr:@std/path@0.213.1", "jsr:@std/path@^0.221.0": "jsr:@std/path@0.221.0", "jsr:@std/streams@^0.223.0": "jsr:@std/streams@0.223.0", - "npm:@electric-sql/pglite@^0.2.5": "npm:@electric-sql/pglite@0.2.5", "npm:@isaacs/ttlcache@^1.4.1": "npm:@isaacs/ttlcache@1.4.1", "npm:@noble/hashes@^1.4.0": "npm:@noble/hashes@1.4.0", "npm:@scure/base@^1.1.6": "npm:@scure/base@1.1.6", "npm:@scure/bip32@^1.4.0": "npm:@scure/bip32@1.4.0", "npm:@scure/bip39@^1.3.0": "npm:@scure/bip39@1.3.0", + "npm:@soapbox.pub/pglite@^0.2.10": "npm:@soapbox.pub/pglite@0.2.10", "npm:@types/node": "npm:@types/node@18.16.19", "npm:comlink-async-generator": "npm:comlink-async-generator@0.0.1", "npm:comlink-async-generator@^0.0.1": "npm:comlink-async-generator@0.0.1", @@ -162,6 +165,13 @@ "jsr:@nostrify/nostrify@^0.22.1" ] }, + "@gleasonator/policy@0.5.0": { + "integrity": "c2882eb3b4147dfe96b6ec2870b012b5a614f686770d1d4b2f778fdc44e8b1f5", + "dependencies": [ + "jsr:@nostrify/nostrify@^0.31.0", + "jsr:@nostrify/policies@^0.33.0" + ] + }, "@hono/hono@4.4.6": { "integrity": "aa557ca9930787ee86b9ca1730691f1ce1c379174c2cb244d5934db2b6314453" }, @@ -257,19 +267,41 @@ "npm:zod@^3.23.8" ] }, + "@nostrify/nostrify@0.31.0": { + "integrity": "1c1b686bb9ca3ad8d19807e3b96ef3793a65d70fd0f433fe6ef8b3fdb9f45557", + "dependencies": [ + "jsr:@nostrify/types@^0.30.1", + "jsr:@std/encoding@^0.224.1", + "npm:@scure/bip32@^1.4.0", + "npm:@scure/bip39@^1.3.0", + "npm:lru-cache@^10.2.0", + "npm:nostr-tools@^2.7.0", + "npm:websocket-ts@^2.1.5", + "npm:zod@^3.23.8" + ] + }, + "@nostrify/policies@0.33.0": { + "integrity": "c946b06d0527298b4d7c9819d142a10f522ba09eee76c37525aa4acfc5d87aee", + "dependencies": [ + "jsr:@nostrify/types@^0.30.1", + "npm:nostr-tools@^2.7.0" + ] + }, "@nostrify/types@0.30.0": { "integrity": "1f38fa849cff930bd709edbf94ef9ac02f46afb8b851f86c8736517b354616da" }, + "@nostrify/types@0.30.1": { + "integrity": "245da176f6893a43250697db51ad32bfa29bf9b1cdc1ca218043d9abf6de5ae5" + }, "@soapbox/kysely-deno-sqlite@2.2.0": { "integrity": "668ec94600bc4b4d7bd618dd7ca65d4ef30ee61c46ffcb379b6f45203c08517a", "dependencies": [ "npm:kysely@^0.27.2" ] }, - "@soapbox/kysely-pglite@0.0.6": { - "integrity": "df88a2610c0bbfe382a0d4d060a1968458b96b79e990e4d7b85744ca819f00d9", + "@soapbox/kysely-pglite@0.0.1": { + "integrity": "7a4221aa780aad6fba9747c45c59dfb1c62017ba8cad9db5607f6e5822c058d5", "dependencies": [ - "jsr:@std/encoding@^1.0.4", "npm:kysely@^0.27.4" ] }, @@ -325,9 +357,6 @@ "@std/encoding@0.224.3": { "integrity": "5e861b6d81be5359fad4155e591acf17c0207b595112d1840998bb9f476dbdaf" }, - "@std/encoding@1.0.4": { - "integrity": "2266cd516b32369e3dc5695717c96bf88343a1f761d6e6187a02a2bbe2af86ae" - }, "@std/fmt@0.213.1": { "integrity": "a06d31777566d874b9c856c10244ac3e6b660bdec4c82506cd46be052a1082c3" }, @@ -422,10 +451,6 @@ } }, "npm": { - "@electric-sql/pglite@0.2.5": { - "integrity": "sha512-LrMX2kX0mCVN4xkhIDv1KBVukWtoOI/+P9MDQgHX5QEeZCi5S60LZOa0VWXjufPEz7mJtbuXWJRujD++t0gsHA==", - "dependencies": {} - }, "@isaacs/ttlcache@1.4.1": { "integrity": "sha512-RQgQ4uQ+pLbqXfOmieB91ejmLwvSgv9nLx6sT6sD83s7umBypgg+OIBOBbEUiJXrfpnp9j0mRhYYdzp9uqq3lA==", "dependencies": {} @@ -510,6 +535,10 @@ "@scure/base": "@scure/base@1.1.6" } }, + "@soapbox.pub/pglite@0.2.10": { + "integrity": "sha512-DEHejCr+R99RNdyOo34Nbl1FKLmpBCc0pMlPhH3yTyc/KH5HV7dPYbTGCgqRXPxODVkQhvaEuIF2266KsUlZcg==", + "dependencies": {} + }, "@types/dompurify@3.0.5": { "integrity": "sha512-1Wg0g3BtQF7sSb27fJQAKck1HECM6zV1EB66j8JH9i3LCjYabJa0FSdiSgsD5K/RbrsR0SiraKacLB+T8ZVYAg==", "dependencies": { @@ -1949,7 +1978,7 @@ "jsr:@nostrify/db@^0.31.2", "jsr:@nostrify/nostrify@^0.30.1", "jsr:@soapbox/kysely-deno-sqlite@^2.1.0", - "jsr:@soapbox/kysely-pglite@^0.0.6", + "jsr:@soapbox/kysely-pglite@^0.0.1", "jsr:@soapbox/stickynotes@^0.4.0", "jsr:@std/assert@^0.225.1", "jsr:@std/cli@^0.223.0", @@ -1960,10 +1989,10 @@ "jsr:@std/json@^0.223.0", "jsr:@std/media-types@^0.224.1", "jsr:@std/streams@^0.223.0", - "npm:@electric-sql/pglite@^0.2.5", "npm:@isaacs/ttlcache@^1.4.1", "npm:@noble/secp256k1@^2.0.0", "npm:@scure/base@^1.1.6", + "npm:@soapbox.pub/pglite@^0.2.10", "npm:comlink-async-generator@^0.0.1", "npm:comlink@^4.4.1", "npm:commander@12.1.0", From 25a8f27cb760311dc32e7c21c26403fa1ad2b5b9 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Tue, 10 Sep 2024 16:01:13 -0500 Subject: [PATCH 31/82] ci: upgrade deno to v1.46.3 --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b97226a0..bb7838bb 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,4 +1,4 @@ -image: denoland/deno:1.45.5 +image: denoland/deno:1.46.3 default: interruptible: true From f76d0af16d33e79a8866894469e161499375b03e Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Wed, 11 Sep 2024 10:28:06 -0500 Subject: [PATCH 32/82] Add TEST_DATABASE_URL env, change database defaults --- src/config.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/config.ts b/src/config.ts index 145ed9e4..a6257e25 100644 --- a/src/config.ts +++ b/src/config.ts @@ -82,7 +82,11 @@ class Conf { * ``` */ static get databaseUrl(): string { - return Deno.env.get('DATABASE_URL') ?? 'pglite://data/pgdata'; + return Deno.env.get('DATABASE_URL') ?? 'file://data/pgdata'; + } + /** Database to use in tests. */ + static get testDatabaseUrl(): string { + return Deno.env.get('TEST_DATABASE_URL') ?? 'memory://'; } static db = { get url(): url.UrlWithStringQuery { From dc8d09a9daf9004d437c21a8aa9c71641ea46336 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Wed, 11 Sep 2024 11:08:33 -0500 Subject: [PATCH 33/82] Remove SQLite support --- .gitlab-ci.yml | 1 - deno.json | 2 - deno.lock | 65 ---------- docs/debugging.md | 6 +- scripts/setup.ts | 8 +- src/config.ts | 31 ----- src/controllers/api/oauth.ts | 2 +- src/controllers/api/statuses.ts | 2 +- src/controllers/api/streaming.ts | 2 +- src/db/DittoDB.ts | 55 ++++----- src/db/DittoTables.ts | 4 +- src/db/adapters/DittoPglite.ts | 33 +----- src/db/adapters/DittoPostgres.ts | 13 +- src/db/adapters/DittoSQLite.ts | 59 ---------- src/db/migrations/002_events_fts.ts | 13 +- src/db/migrations/019_ndatabase_schema.ts | 14 +-- src/middleware/signerMiddleware.ts | 2 +- src/pipeline.ts | 8 +- src/storages.ts | 2 +- src/storages/EventsDB.ts | 29 ++--- src/storages/hydrate.ts | 2 +- src/test.ts | 137 ++++++---------------- src/trends.ts | 113 ++++++------------ src/utils/SimpleLRU.ts | 2 +- src/utils/stats.ts | 23 ++-- src/workers/fetch.worker.ts | 2 +- src/workers/sqlite.ts | 52 -------- src/workers/sqlite.worker.ts | 42 ------- 28 files changed, 156 insertions(+), 568 deletions(-) delete mode 100644 src/db/adapters/DittoSQLite.ts delete mode 100644 src/workers/sqlite.ts delete mode 100644 src/workers/sqlite.worker.ts diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index bb7838bb..a64b111d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -42,4 +42,3 @@ postgres: DITTO_NSEC: nsec1zyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zygs4rm7hz DATABASE_URL: postgres://postgres:postgres@postgres:5432/postgres POSTGRES_HOST_AUTH_METHOD: trust - ALLOW_TO_USE_DATABASE_URL: true diff --git a/deno.json b/deno.json index 840caf77..4897cff4 100644 --- a/deno.json +++ b/deno.json @@ -27,7 +27,6 @@ "@/": "./src/", "@b-fuze/deno-dom": "jsr:@b-fuze/deno-dom@^0.1.47", "@bradenmacdonald/s3-lite-client": "jsr:@bradenmacdonald/s3-lite-client@^0.7.4", - "@db/sqlite": "jsr:@db/sqlite@^0.11.1", "@electric-sql/pglite": "npm:@soapbox.pub/pglite@^0.2.10", "@hono/hono": "jsr:@hono/hono@^4.4.6", "@isaacs/ttlcache": "npm:@isaacs/ttlcache@^1.4.1", @@ -37,7 +36,6 @@ "@nostrify/nostrify": "jsr:@nostrify/nostrify@^0.30.1", "@scure/base": "npm:@scure/base@^1.1.6", "@sentry/deno": "https://deno.land/x/sentry@7.112.2/index.mjs", - "@soapbox/kysely-deno-sqlite": "jsr:@soapbox/kysely-deno-sqlite@^2.1.0", "@soapbox/kysely-pglite": "jsr:@soapbox/kysely-pglite@^0.0.1", "@soapbox/stickynotes": "jsr:@soapbox/stickynotes@^0.4.0", "@std/assert": "jsr:@std/assert@^0.225.1", diff --git a/deno.lock b/deno.lock index d9eee2d8..ca1f8be3 100644 --- a/deno.lock +++ b/deno.lock @@ -4,8 +4,6 @@ "specifiers": { "jsr:@b-fuze/deno-dom@^0.1.47": "jsr:@b-fuze/deno-dom@0.1.48", "jsr:@bradenmacdonald/s3-lite-client@^0.7.4": "jsr:@bradenmacdonald/s3-lite-client@0.7.6", - "jsr:@db/sqlite@^0.11.1": "jsr:@db/sqlite@0.11.1", - "jsr:@denosaurs/plug@1": "jsr:@denosaurs/plug@1.0.6", "jsr:@denosaurs/plug@1.0.3": "jsr:@denosaurs/plug@1.0.3", "jsr:@gleasonator/policy": "jsr:@gleasonator/policy@0.2.0", "jsr:@gleasonator/policy@0.2.0": "jsr:@gleasonator/policy@0.2.0", @@ -25,12 +23,9 @@ "jsr:@nostrify/policies@^0.33.0": "jsr:@nostrify/policies@0.33.0", "jsr:@nostrify/types@^0.30.0": "jsr:@nostrify/types@0.30.1", "jsr:@nostrify/types@^0.30.1": "jsr:@nostrify/types@0.30.1", - "jsr:@soapbox/kysely-deno-sqlite@^2.1.0": "jsr:@soapbox/kysely-deno-sqlite@2.2.0", "jsr:@soapbox/kysely-pglite@^0.0.1": "jsr:@soapbox/kysely-pglite@0.0.1", "jsr:@soapbox/stickynotes@^0.4.0": "jsr:@soapbox/stickynotes@0.4.0", "jsr:@std/assert@^0.213.1": "jsr:@std/assert@0.213.1", - "jsr:@std/assert@^0.217.0": "jsr:@std/assert@0.217.0", - "jsr:@std/assert@^0.221.0": "jsr:@std/assert@0.221.0", "jsr:@std/assert@^0.224.0": "jsr:@std/assert@0.224.0", "jsr:@std/assert@^0.225.1": "jsr:@std/assert@0.225.3", "jsr:@std/bytes@^0.224.0": "jsr:@std/bytes@0.224.0", @@ -41,22 +36,17 @@ "jsr:@std/crypto@^0.224.0": "jsr:@std/crypto@0.224.0", "jsr:@std/dotenv@^0.224.0": "jsr:@std/dotenv@0.224.2", "jsr:@std/encoding@0.213.1": "jsr:@std/encoding@0.213.1", - "jsr:@std/encoding@^0.221.0": "jsr:@std/encoding@0.221.0", "jsr:@std/encoding@^0.224.0": "jsr:@std/encoding@0.224.3", "jsr:@std/encoding@^0.224.1": "jsr:@std/encoding@0.224.3", "jsr:@std/fmt@0.213.1": "jsr:@std/fmt@0.213.1", - "jsr:@std/fmt@^0.221.0": "jsr:@std/fmt@0.221.0", "jsr:@std/fs@0.213.1": "jsr:@std/fs@0.213.1", - "jsr:@std/fs@^0.221.0": "jsr:@std/fs@0.221.0", "jsr:@std/fs@^0.229.3": "jsr:@std/fs@0.229.3", "jsr:@std/internal@^1.0.0": "jsr:@std/internal@1.0.1", "jsr:@std/io@^0.224": "jsr:@std/io@0.224.7", "jsr:@std/json@^0.223.0": "jsr:@std/json@0.223.0", "jsr:@std/media-types@^0.224.1": "jsr:@std/media-types@0.224.1", "jsr:@std/path@0.213.1": "jsr:@std/path@0.213.1", - "jsr:@std/path@0.217": "jsr:@std/path@0.217.0", "jsr:@std/path@^0.213.1": "jsr:@std/path@0.213.1", - "jsr:@std/path@^0.221.0": "jsr:@std/path@0.221.0", "jsr:@std/streams@^0.223.0": "jsr:@std/streams@0.223.0", "npm:@isaacs/ttlcache@^1.4.1": "npm:@isaacs/ttlcache@1.4.1", "npm:@noble/hashes@^1.4.0": "npm:@noble/hashes@1.4.0", @@ -116,13 +106,6 @@ "jsr:@std/io@^0.224" ] }, - "@db/sqlite@0.11.1": { - "integrity": "546434e7ed762db07e6ade0f963540dd5e06723b802937bf260ff855b21ef9c5", - "dependencies": [ - "jsr:@denosaurs/plug@1", - "jsr:@std/path@0.217" - ] - }, "@denosaurs/plug@1.0.3": { "integrity": "b010544e386bea0ff3a1d05e0c88f704ea28cbd4d753439c2f1ee021a85d4640", "dependencies": [ @@ -132,15 +115,6 @@ "jsr:@std/path@0.213.1" ] }, - "@denosaurs/plug@1.0.6": { - "integrity": "6cf5b9daba7799837b9ffbe89f3450510f588fafef8115ddab1ff0be9cb7c1a7", - "dependencies": [ - "jsr:@std/encoding@^0.221.0", - "jsr:@std/fmt@^0.221.0", - "jsr:@std/fs@^0.221.0", - "jsr:@std/path@^0.221.0" - ] - }, "@gleasonator/policy@0.2.0": { "integrity": "3fe58b853ab203b2b67e65b64391dbcf5c07bc1caaf46e97b2f8ed5b14f30fdf", "dependencies": [ @@ -293,12 +267,6 @@ "@nostrify/types@0.30.1": { "integrity": "245da176f6893a43250697db51ad32bfa29bf9b1cdc1ca218043d9abf6de5ae5" }, - "@soapbox/kysely-deno-sqlite@2.2.0": { - "integrity": "668ec94600bc4b4d7bd618dd7ca65d4ef30ee61c46ffcb379b6f45203c08517a", - "dependencies": [ - "npm:kysely@^0.27.2" - ] - }, "@soapbox/kysely-pglite@0.0.1": { "integrity": "7a4221aa780aad6fba9747c45c59dfb1c62017ba8cad9db5607f6e5822c058d5", "dependencies": [ @@ -311,12 +279,6 @@ "@std/assert@0.213.1": { "integrity": "24c28178b30c8e0782c18e8e94ea72b16282207569cdd10ffb9d1d26f2edebfe" }, - "@std/assert@0.217.0": { - "integrity": "c98e279362ca6982d5285c3b89517b757c1e3477ee9f14eb2fdf80a45aaa9642" - }, - "@std/assert@0.221.0": { - "integrity": "a5f1aa6e7909dbea271754fd4ab3f4e687aeff4873b4cef9a320af813adb489a" - }, "@std/assert@0.224.0": { "integrity": "8643233ec7aec38a940a8264a6e3eed9bfa44e7a71cc6b3c8874213ff401967f" }, @@ -351,18 +313,12 @@ "@std/encoding@0.213.1": { "integrity": "fcbb6928713dde941a18ca5db88ca1544d0755ec8fb20fe61e2dc8144b390c62" }, - "@std/encoding@0.221.0": { - "integrity": "d1dd76ef0dc5d14088411e6dc1dede53bf8308c95d1537df1214c97137208e45" - }, "@std/encoding@0.224.3": { "integrity": "5e861b6d81be5359fad4155e591acf17c0207b595112d1840998bb9f476dbdaf" }, "@std/fmt@0.213.1": { "integrity": "a06d31777566d874b9c856c10244ac3e6b660bdec4c82506cd46be052a1082c3" }, - "@std/fmt@0.221.0": { - "integrity": "379fed69bdd9731110f26b9085aeb740606b20428ce6af31ef6bd45ef8efa62a" - }, "@std/fs@0.213.1": { "integrity": "fbcaf099f8a85c27ab0712b666262cda8fe6d02e9937bf9313ecaea39a22c501", "dependencies": [ @@ -370,13 +326,6 @@ "jsr:@std/path@^0.213.1" ] }, - "@std/fs@0.221.0": { - "integrity": "028044450299de8ed5a716ade4e6d524399f035513b85913794f4e81f07da286", - "dependencies": [ - "jsr:@std/assert@^0.221.0", - "jsr:@std/path@^0.221.0" - ] - }, "@std/fs@0.229.3": { "integrity": "783bca21f24da92e04c3893c9e79653227ab016c48e96b3078377ebd5222e6eb" }, @@ -434,18 +383,6 @@ "jsr:@std/assert@^0.213.1" ] }, - "@std/path@0.217.0": { - "integrity": "1217cc25534bca9a2f672d7fe7c6f356e4027df400c0e85c0ef3e4343bc67d11", - "dependencies": [ - "jsr:@std/assert@^0.217.0" - ] - }, - "@std/path@0.221.0": { - "integrity": "0a36f6b17314ef653a3a1649740cc8db51b25a133ecfe838f20b79a56ebe0095", - "dependencies": [ - "jsr:@std/assert@^0.221.0" - ] - }, "@std/streams@0.223.0": { "integrity": "d6b28e498ced3960b04dc5d251f2dcfc1df244b5ec5a48dc23a8f9b490be3b99" } @@ -1972,12 +1909,10 @@ "dependencies": [ "jsr:@b-fuze/deno-dom@^0.1.47", "jsr:@bradenmacdonald/s3-lite-client@^0.7.4", - "jsr:@db/sqlite@^0.11.1", "jsr:@hono/hono@^4.4.6", "jsr:@lambdalisue/async@^2.1.1", "jsr:@nostrify/db@^0.31.2", "jsr:@nostrify/nostrify@^0.30.1", - "jsr:@soapbox/kysely-deno-sqlite@^2.1.0", "jsr:@soapbox/kysely-pglite@^0.0.1", "jsr:@soapbox/stickynotes@^0.4.0", "jsr:@std/assert@^0.225.1", diff --git a/docs/debugging.md b/docs/debugging.md index 6abc513d..879f36cd 100644 --- a/docs/debugging.md +++ b/docs/debugging.md @@ -16,12 +16,12 @@ ssh -L 9229:localhost:9229 @ Then, in Chromium, go to `chrome://inspect` and the Ditto server should be available. -## SQLite performance +## SQL performance -To track slow queries, first set `DEBUG=ditto:sqlite.worker` in the environment so only SQLite logs are shown. +To track slow queries, first set `DEBUG=ditto:sql` in the environment so only SQL logs are shown. Then, grep for any logs above 0.001s: ```sh journalctl -fu ditto | grep -v '(0.00s)' -``` \ No newline at end of file +``` diff --git a/scripts/setup.ts b/scripts/setup.ts index 9a6d6f34..1365fec5 100644 --- a/scripts/setup.ts +++ b/scripts/setup.ts @@ -45,10 +45,10 @@ const DATABASE_URL = Deno.env.get('DATABASE_URL'); if (DATABASE_URL) { vars.DATABASE_URL = await question('input', 'Database URL', DATABASE_URL); } else { - const database = await question('list', 'Which database do you want to use?', ['postgres', 'sqlite']); - if (database === 'sqlite') { - const path = await question('input', 'Path to SQLite database', 'data/db.sqlite3'); - vars.DATABASE_URL = `sqlite://${path}`; + const database = await question('list', 'Which database do you want to use?', ['postgres', 'pglite']); + if (database === 'pglite') { + const path = await question('input', 'Path to PGlite data directory', 'data/pgdata'); + vars.DATABASE_URL = `file://${path}`; } if (database === 'postgres') { const host = await question('input', 'Postgres host', 'localhost'); diff --git a/src/config.ts b/src/config.ts index a6257e25..5fa7be9a 100644 --- a/src/config.ts +++ b/src/config.ts @@ -1,5 +1,3 @@ -import url from 'node:url'; - import * as dotenv from '@std/dotenv'; import { getPublicKey, nip19 } from 'nostr-tools'; import { z } from 'zod'; @@ -89,20 +87,6 @@ class Conf { return Deno.env.get('TEST_DATABASE_URL') ?? 'memory://'; } static db = { - get url(): url.UrlWithStringQuery { - return url.parse(Conf.databaseUrl); - }, - get dialect(): 'sqlite' | 'postgres' | undefined { - switch (Conf.db.url.protocol) { - case 'sqlite:': - return 'sqlite'; - case 'pglite:': - case 'postgres:': - case 'postgresql:': - return 'postgres'; - } - return undefined; - }, /** Database query timeout configurations. */ timeouts: { /** Default query timeout when another setting isn't more specific. */ @@ -221,21 +205,6 @@ class Conf { static get sentryDsn(): string | undefined { return Deno.env.get('SENTRY_DSN'); } - /** SQLite settings. */ - static sqlite = { - /** - * Number of bytes to use for memory-mapped IO. - * https://www.sqlite.org/pragma.html#pragma_mmap_size - */ - get mmapSize(): number { - const value = Deno.env.get('SQLITE_MMAP_SIZE'); - if (value) { - return Number(value); - } else { - return 1024 * 1024 * 1024; - } - }, - }; /** Postgres settings. */ static pg = { /** Number of connections to use in the pool. */ diff --git a/src/controllers/api/oauth.ts b/src/controllers/api/oauth.ts index a736f5ca..01f80bf1 100644 --- a/src/controllers/api/oauth.ts +++ b/src/controllers/api/oauth.ts @@ -82,7 +82,7 @@ const createTokenController: AppController = async (c) => { async function getToken( { pubkey, secret, relays = [] }: { pubkey: string; secret?: string; relays?: string[] }, ): Promise<`token1${string}`> { - const { kysely } = await DittoDB.getInstance(); + const kysely = await DittoDB.getInstance(); const token = generateToken(); const serverSeckey = generateSecretKey(); diff --git a/src/controllers/api/statuses.ts b/src/controllers/api/statuses.ts index 05b5022c..4236db52 100644 --- a/src/controllers/api/statuses.ts +++ b/src/controllers/api/statuses.ts @@ -578,7 +578,7 @@ const zappedByController: AppController = async (c) => { const id = c.req.param('id'); const params = c.get('listPagination'); const store = await Storages.db(); - const { kysely } = await DittoDB.getInstance(); + const kysely = await DittoDB.getInstance(); const zaps = await kysely.selectFrom('event_zaps') .selectAll() diff --git a/src/controllers/api/streaming.ts b/src/controllers/api/streaming.ts index 047aa573..f70a6b06 100644 --- a/src/controllers/api/streaming.ts +++ b/src/controllers/api/streaming.ts @@ -222,7 +222,7 @@ async function topicToFilter( async function getTokenPubkey(token: string): Promise { if (token.startsWith('token1')) { - const { kysely } = await DittoDB.getInstance(); + const kysely = await DittoDB.getInstance(); const { user_pubkey } = await kysely .selectFrom('nip46_tokens') diff --git a/src/db/DittoDB.ts b/src/db/DittoDB.ts index 40d03e5f..03638a98 100644 --- a/src/db/DittoDB.ts +++ b/src/db/DittoDB.ts @@ -1,75 +1,66 @@ import fs from 'node:fs/promises'; import path from 'node:path'; -import { NDatabaseSchema, NPostgresSchema } from '@nostrify/db'; import { FileMigrationProvider, Kysely, Migrator } from 'kysely'; import { Conf } from '@/config.ts'; import { DittoPglite } from '@/db/adapters/DittoPglite.ts'; import { DittoPostgres } from '@/db/adapters/DittoPostgres.ts'; -import { DittoSQLite } from '@/db/adapters/DittoSQLite.ts'; import { DittoTables } from '@/db/DittoTables.ts'; -export type DittoDatabase = { - dialect: 'sqlite'; - kysely: Kysely & Kysely; -} | { - dialect: 'postgres'; - kysely: Kysely & Kysely; -}; - export class DittoDB { - private static db: Promise | undefined; + private static kysely: Promise> | undefined; - static getInstance(): Promise { - if (!this.db) { - this.db = this._getInstance(); + static getInstance(): Promise> { + if (!this.kysely) { + this.kysely = this._getInstance(); } - return this.db; + return this.kysely; } - static async _getInstance(): Promise { - const result = {} as DittoDatabase; + static async _getInstance(): Promise> { + const { protocol } = new URL(Conf.databaseUrl); - switch (Conf.db.url.protocol) { - case 'sqlite:': - result.dialect = 'sqlite'; - result.kysely = await DittoSQLite.getInstance(); - break; - case 'pglite:': - result.dialect = 'postgres'; - result.kysely = await DittoPglite.getInstance(); + let kysely: Kysely; + + switch (protocol) { + case 'file:': + case 'memory:': + kysely = await DittoPglite.getInstance(); break; case 'postgres:': case 'postgresql:': - result.dialect = 'postgres'; - result.kysely = await DittoPostgres.getInstance(); + kysely = await DittoPostgres.getInstance(); break; default: throw new Error('Unsupported database URL.'); } - await this.migrate(result.kysely); + await this.migrate(kysely); - return result; + return kysely; } static get poolSize(): number { - if (Conf.db.dialect === 'postgres') { + const { protocol } = new URL(Conf.databaseUrl); + + if (['postgres:', 'postgresql:'].includes(protocol)) { return DittoPostgres.poolSize; } return 1; } static get availableConnections(): number { - if (Conf.db.dialect === 'postgres') { + const { protocol } = new URL(Conf.databaseUrl); + + if (['postgres:', 'postgresql:'].includes(protocol)) { return DittoPostgres.availableConnections; } return 1; } /** Migrate the database to the latest version. */ - static async migrate(kysely: DittoDatabase['kysely']) { + static async migrate(kysely: Kysely) { const migrator = new Migrator({ db: kysely, provider: new FileMigrationProvider({ diff --git a/src/db/DittoTables.ts b/src/db/DittoTables.ts index 09bf3e43..a62c485d 100644 --- a/src/db/DittoTables.ts +++ b/src/db/DittoTables.ts @@ -1,4 +1,6 @@ -export interface DittoTables { +import { NPostgresSchema } from '@nostrify/db'; + +export interface DittoTables extends NPostgresSchema { nip46_tokens: NIP46TokenRow; author_stats: AuthorStatsRow; event_stats: EventStatsRow; diff --git a/src/db/adapters/DittoPglite.ts b/src/db/adapters/DittoPglite.ts index ef035a3b..80ea6087 100644 --- a/src/db/adapters/DittoPglite.ts +++ b/src/db/adapters/DittoPglite.ts @@ -1,5 +1,4 @@ import { PGlite } from '@electric-sql/pglite'; -import { NPostgresSchema } from '@nostrify/db'; import { PgliteDialect } from '@soapbox/kysely-pglite'; import { Kysely } from 'kysely'; @@ -8,17 +7,17 @@ import { DittoTables } from '@/db/DittoTables.ts'; import { KyselyLogger } from '@/db/KyselyLogger.ts'; export class DittoPglite { - static db: Kysely & Kysely | undefined; + static db: Kysely | undefined; // deno-lint-ignore require-await - static async getInstance(): Promise & Kysely> { + static async getInstance(): Promise> { if (!this.db) { - this.db = new Kysely({ + this.db = new Kysely({ dialect: new PgliteDialect({ - database: new PGlite(this.path), + database: new PGlite(Conf.databaseUrl), }), log: KyselyLogger, - }) as Kysely & Kysely; + }) as Kysely; } return this.db; @@ -31,26 +30,4 @@ export class DittoPglite { static get availableConnections(): number { return 1; } - - /** Get the relative or absolute path based on the `DATABASE_URL`. */ - static get path(): string | undefined { - if (Conf.databaseUrl === 'pglite://:memory:') { - return undefined; - } - - const { host, pathname } = Conf.db.url; - - if (!pathname) return ''; - - // Get relative path. - if (host === '') { - return pathname; - } else if (host === '.') { - return pathname; - } else if (host) { - return host + pathname; - } - - return ''; - } } diff --git a/src/db/adapters/DittoPostgres.ts b/src/db/adapters/DittoPostgres.ts index d1127117..b0a6c93a 100644 --- a/src/db/adapters/DittoPostgres.ts +++ b/src/db/adapters/DittoPostgres.ts @@ -1,4 +1,3 @@ -import { NPostgresSchema } from '@nostrify/db'; import { BinaryOperationNode, FunctionNode, @@ -18,17 +17,17 @@ import { DittoTables } from '@/db/DittoTables.ts'; import { KyselyLogger } from '@/db/KyselyLogger.ts'; export class DittoPostgres { - static db: Kysely & Kysely | undefined; + static kysely: Kysely | undefined; static postgres?: postgres.Sql; // deno-lint-ignore require-await - static async getInstance(): Promise & Kysely> { + static async getInstance(): Promise> { if (!this.postgres) { this.postgres = postgres(Conf.databaseUrl, { max: Conf.pg.poolSize }); } - if (!this.db) { - this.db = new Kysely({ + if (!this.kysely) { + this.kysely = new Kysely({ dialect: { createAdapter() { return new PostgresAdapter(); @@ -46,10 +45,10 @@ export class DittoPostgres { }, }, log: KyselyLogger, - }) as Kysely & Kysely; + }); } - return this.db; + return this.kysely; } static get poolSize() { diff --git a/src/db/adapters/DittoSQLite.ts b/src/db/adapters/DittoSQLite.ts deleted file mode 100644 index e54292dd..00000000 --- a/src/db/adapters/DittoSQLite.ts +++ /dev/null @@ -1,59 +0,0 @@ -import { NDatabaseSchema } from '@nostrify/db'; -import { PolySqliteDialect } from '@soapbox/kysely-deno-sqlite'; -import { Kysely, sql } from 'kysely'; - -import { Conf } from '@/config.ts'; -import { DittoTables } from '@/db/DittoTables.ts'; -import { KyselyLogger } from '@/db/KyselyLogger.ts'; -import SqliteWorker from '@/workers/sqlite.ts'; - -export class DittoSQLite { - static db: Kysely & Kysely | undefined; - - static async getInstance(): Promise & Kysely> { - if (!this.db) { - const sqliteWorker = new SqliteWorker(); - await sqliteWorker.open(this.path); - - this.db = new Kysely({ - dialect: new PolySqliteDialect({ - database: sqliteWorker, - }), - log: KyselyLogger, - }) as Kysely & Kysely; - - // Set PRAGMA values. - await Promise.all([ - sql`PRAGMA synchronous = normal`.execute(this.db), - sql`PRAGMA temp_store = memory`.execute(this.db), - sql`PRAGMA foreign_keys = ON`.execute(this.db), - sql`PRAGMA auto_vacuum = FULL`.execute(this.db), - sql`PRAGMA journal_mode = WAL`.execute(this.db), - sql.raw(`PRAGMA mmap_size = ${Conf.sqlite.mmapSize}`).execute(this.db), - ]); - } - return this.db; - } - - /** Get the relative or absolute path based on the `DATABASE_URL`. */ - static get path() { - if (Conf.databaseUrl === 'sqlite://:memory:') { - return ':memory:'; - } - - const { host, pathname } = Conf.db.url; - - if (!pathname) return ''; - - // Get relative path. - if (host === '') { - return pathname; - } else if (host === '.') { - return pathname; - } else if (host) { - return host + pathname; - } - - return ''; - } -} diff --git a/src/db/migrations/002_events_fts.ts b/src/db/migrations/002_events_fts.ts index 56abab5f..45ad03e4 100644 --- a/src/db/migrations/002_events_fts.ts +++ b/src/db/migrations/002_events_fts.ts @@ -1,13 +1,8 @@ -import { Kysely, sql } from 'kysely'; +import { Kysely } from 'kysely'; -import { Conf } from '@/config.ts'; - -export async function up(db: Kysely): Promise { - if (Conf.db.dialect === 'sqlite') { - await sql`CREATE VIRTUAL TABLE events_fts USING fts5(id, content)`.execute(db); - } +export async function up(_db: Kysely): Promise { + // This migration used to create an FTS table for SQLite, but SQLite support was removed. } -export async function down(db: Kysely): Promise { - await db.schema.dropTable('events_fts').ifExists().execute(); +export async function down(_db: Kysely): Promise { } diff --git a/src/db/migrations/019_ndatabase_schema.ts b/src/db/migrations/019_ndatabase_schema.ts index 31b86cd3..79d8cbc9 100644 --- a/src/db/migrations/019_ndatabase_schema.ts +++ b/src/db/migrations/019_ndatabase_schema.ts @@ -1,25 +1,13 @@ -import { Kysely, sql } from 'kysely'; - -import { Conf } from '@/config.ts'; +import { Kysely } from 'kysely'; export async function up(db: Kysely): Promise { await db.schema.alterTable('events').renameTo('nostr_events').execute(); await db.schema.alterTable('tags').renameTo('nostr_tags').execute(); await db.schema.alterTable('nostr_tags').renameColumn('tag', 'name').execute(); - - if (Conf.db.dialect === 'sqlite') { - await db.schema.dropTable('events_fts').execute(); - await sql`CREATE VIRTUAL TABLE nostr_fts5 USING fts5(event_id, content)`.execute(db); - } } export async function down(db: Kysely): Promise { await db.schema.alterTable('nostr_events').renameTo('events').execute(); await db.schema.alterTable('nostr_tags').renameTo('tags').execute(); await db.schema.alterTable('tags').renameColumn('name', 'tag').execute(); - - if (Conf.db.dialect === 'sqlite') { - await db.schema.dropTable('nostr_fts5').execute(); - await sql`CREATE VIRTUAL TABLE events_fts USING fts5(id, content)`.execute(db); - } } diff --git a/src/middleware/signerMiddleware.ts b/src/middleware/signerMiddleware.ts index 60826db9..89a494c4 100644 --- a/src/middleware/signerMiddleware.ts +++ b/src/middleware/signerMiddleware.ts @@ -20,7 +20,7 @@ export const signerMiddleware: AppMiddleware = async (c, next) => { if (bech32.startsWith('token1')) { try { - const { kysely } = await DittoDB.getInstance(); + const kysely = await DittoDB.getInstance(); const { user_pubkey, server_seckey, relays } = await kysely .selectFrom('nip46_tokens') diff --git a/src/pipeline.ts b/src/pipeline.ts index dd59cb8d..ceb370ca 100644 --- a/src/pipeline.ts +++ b/src/pipeline.ts @@ -53,7 +53,7 @@ async function handleEvent(event: DittoEvent, signal: AbortSignal): Promise { async function hydrateEvent(event: DittoEvent, signal: AbortSignal): Promise { await hydrateEvents({ events: [event], store: await Storages.db(), signal }); - const { kysely } = await DittoDB.getInstance(); + const kysely = await DittoDB.getInstance(); const domain = await kysely .selectFrom('pubkey_domains') .select('domain') @@ -118,7 +118,7 @@ async function hydrateEvent(event: DittoEvent, signal: AbortSignal): Promise { if (NKinds.ephemeral(event.kind)) return; const store = await Storages.db(); - const { kysely } = await DittoDB.getInstance(); + const kysely = await DittoDB.getInstance(); await updateStats({ event, store, kysely }).catch(debug); await store.event(event, { signal }); @@ -146,7 +146,7 @@ async function parseMetadata(event: NostrEvent, signal: AbortSignal): Promise | undefined; private static _search: Promise | undefined; - /** SQLite database to store events this Ditto server cares about. */ + /** SQL database to store events this Ditto server cares about. */ public static async db(): Promise { if (!this._db) { this._db = (async () => { diff --git a/src/storages/EventsDB.ts b/src/storages/EventsDB.ts index 7cbd9c22..6c006dc2 100644 --- a/src/storages/EventsDB.ts +++ b/src/storages/EventsDB.ts @@ -13,10 +13,11 @@ import { NStore, } from '@nostrify/nostrify'; import { Stickynotes } from '@soapbox/stickynotes'; +import { Kysely } from 'kysely'; import { nip27 } from 'nostr-tools'; import { Conf } from '@/config.ts'; -import { DittoDatabase } from '@/db/DittoDB.ts'; +import { DittoTables } from '@/db/DittoTables.ts'; import { dbEventsCounter } from '@/metrics.ts'; import { RelayError } from '@/RelayError.ts'; import { purifyEvent } from '@/storages/hydrate.ts'; @@ -30,7 +31,7 @@ type TagCondition = ({ event, count, value }: { value: string; }) => boolean; -/** SQLite database storage adapter for Nostr events. */ +/** SQL database storage adapter for Nostr events. */ class EventsDB implements NStore { private store: NDatabase | NPostgres; private console = new Stickynotes('ditto:db:events'); @@ -52,21 +53,11 @@ class EventsDB implements NStore { 't': ({ event, count, value }) => (event.kind === 1985 ? count < 20 : count < 5) && value.length < 50, }; - constructor(private database: DittoDatabase) { - const { dialect, kysely } = database; - - if (dialect === 'postgres') { - this.store = new NPostgres(kysely, { - indexTags: EventsDB.indexTags, - indexSearch: EventsDB.searchText, - }); - } else { - this.store = new NDatabase(kysely, { - fts: 'sqlite', - indexTags: EventsDB.indexTags, - searchText: EventsDB.searchText, - }); - } + constructor(private kysely: Kysely) { + this.store = new NPostgres(kysely, { + indexTags: EventsDB.indexTags, + indexSearch: EventsDB.searchText, + }); } /** Insert an event (and its tags) into the database. */ @@ -273,7 +264,7 @@ class EventsDB implements NStore { return tags.map(([_tag, value]) => value).join('\n'); } - /** Converts filters to more performant, simpler filters that are better for SQLite. */ + /** Converts filters to more performant, simpler filters. */ async expandFilters(filters: NostrFilter[]): Promise { filters = structuredClone(filters); @@ -286,7 +277,7 @@ class EventsDB implements NStore { ) as { key: 'domain'; value: string } | undefined)?.value; if (domain) { - const query = this.database.kysely + const query = this.kysely .selectFrom('pubkey_domains') .select('pubkey') .where('domain', '=', domain); diff --git a/src/storages/hydrate.ts b/src/storages/hydrate.ts index 19ba0db4..2294f880 100644 --- a/src/storages/hydrate.ts +++ b/src/storages/hydrate.ts @@ -18,7 +18,7 @@ interface HydrateOpts { /** Hydrate events using the provided storage. */ async function hydrateEvents(opts: HydrateOpts): Promise { - const { events, store, signal, kysely = (await DittoDB.getInstance()).kysely } = opts; + const { events, store, signal, kysely = await DittoDB.getInstance() } = opts; if (!events.length) { return events; diff --git a/src/test.ts b/src/test.ts index df6c84f6..3f144eb6 100644 --- a/src/test.ts +++ b/src/test.ts @@ -1,21 +1,17 @@ -import fs from 'node:fs/promises'; -import path from 'node:path'; - -import { Database as Sqlite } from '@db/sqlite'; -import { NDatabase, NDatabaseSchema, NPostgresSchema } from '@nostrify/db'; +import { PGlite } from '@electric-sql/pglite'; import { NostrEvent } from '@nostrify/nostrify'; -import { DenoSqlite3Dialect } from '@soapbox/kysely-deno-sqlite'; +import { PgliteDialect } from '@soapbox/kysely-pglite'; import { finalizeEvent, generateSecretKey } from 'nostr-tools'; -import { FileMigrationProvider, Kysely, Migrator } from 'kysely'; +import { Kysely } from 'kysely'; import { PostgresJSDialect, PostgresJSDialectConfig } from 'kysely-postgres-js'; import postgres from 'postgres'; -import { DittoDatabase, DittoDB } from '@/db/DittoDB.ts'; +import { Conf } from '@/config.ts'; +import { DittoDB } from '@/db/DittoDB.ts'; import { DittoTables } from '@/db/DittoTables.ts'; import { purifyEvent } from '@/storages/hydrate.ts'; import { KyselyLogger } from '@/db/KyselyLogger.ts'; import { EventsDB } from '@/storages/EventsDB.ts'; -import { Conf } from '@/config.ts'; /** Import an event fixture by name in tests. */ export async function eventFixture(name: string): Promise { @@ -42,97 +38,45 @@ export function genEvent(t: Partial = {}, sk: Uint8Array = generateS return purifyEvent(event); } -/** Get an in-memory SQLite database to use for testing. It's automatically destroyed when it goes out of scope. */ -export async function getTestDB() { - const kysely = new Kysely({ - dialect: new DenoSqlite3Dialect({ - database: new Sqlite(':memory:'), - }), - }); +/** Create an database for testing. */ +export const createTestDB = async (databaseUrl = Conf.testDatabaseUrl) => { + const { protocol } = new URL(databaseUrl); - const migrator = new Migrator({ - db: kysely, - provider: new FileMigrationProvider({ - fs, - path, - migrationFolder: new URL(import.meta.resolve('./db/migrations')).pathname, - }), - }); + const kysely: Kysely = (() => { + switch (protocol) { + case 'postgres:': + case 'postgresql:': + return new Kysely({ + // @ts-ignore Kysely version mismatch. + dialect: new PostgresJSDialect({ + postgres: postgres(databaseUrl, { + max: Conf.pg.poolSize, + }) as unknown as PostgresJSDialectConfig['postgres'], + }), + log: KyselyLogger, + }); + case 'file:': + case 'memory:': + return new Kysely({ + dialect: new PgliteDialect({ + database: new PGlite(databaseUrl), + }), + }); + default: + throw new Error(`Unsupported database URL protocol: ${protocol}`); + } + })(); - await migrator.migrateToLatest(); - - const store = new NDatabase(kysely); + await DittoDB.migrate(kysely); + const store = new EventsDB(kysely); return { store, kysely, - [Symbol.asyncDispose]: () => kysely.destroy(), - }; -} - -/** Create an database for testing. */ -export const createTestDB = async (databaseUrl?: string) => { - databaseUrl ??= Deno.env.get('DATABASE_URL') ?? 'sqlite://:memory:'; - - let dialect: 'sqlite' | 'postgres' = (() => { - const protocol = databaseUrl.split(':')[0]; - switch (protocol) { - case 'sqlite': - return 'sqlite'; - case 'postgres': - return protocol; - case 'postgresql': - return 'postgres'; - default: - throw new Error(`Unsupported protocol: ${protocol}`); - } - })(); - - const allowToUseDATABASE_URL = Deno.env.get('ALLOW_TO_USE_DATABASE_URL')?.toLowerCase() ?? ''; - if (allowToUseDATABASE_URL !== 'true' && dialect === 'postgres') { - console.warn( - '%cRunning tests with sqlite, if you meant to use Postgres, run again with ALLOW_TO_USE_DATABASE_URL environment variable set to true', - 'color: yellow;', - ); - dialect = 'sqlite'; - } - - console.warn(`Using: ${dialect}`); - - const db: DittoDatabase = { dialect } as DittoDatabase; - - if (dialect === 'sqlite') { - // migration 021_pgfts_index.ts calls 'Conf.db.dialect', - // and this calls the DATABASE_URL environment variable. - // The following line ensures to NOT use the DATABASE_URL that may exist in an .env file. - Deno.env.set('DATABASE_URL', 'sqlite://:memory:'); - - db.kysely = new Kysely({ - dialect: new DenoSqlite3Dialect({ - database: new Sqlite(':memory:'), - }), - }) as Kysely & Kysely; - } else { - db.kysely = new Kysely({ - // @ts-ignore Kysely version mismatch. - dialect: new PostgresJSDialect({ - postgres: postgres(Conf.databaseUrl, { - max: Conf.pg.poolSize, - }) as unknown as PostgresJSDialectConfig['postgres'], - }), - log: KyselyLogger, - }) as Kysely & Kysely; - } - - await DittoDB.migrate(db.kysely); - const store = new EventsDB(db); - - return { - dialect, - store, - kysely: db.kysely, [Symbol.asyncDispose]: async () => { - if (dialect === 'postgres') { + // If we're testing against real Postgres, we will reuse the database + // between tests, so we should drop the tables to keep each test fresh. + if (['postgres:', 'postgresql:'].includes(protocol)) { for ( const table of [ 'author_stats', @@ -142,16 +86,13 @@ export const createTestDB = async (databaseUrl?: string) => { 'kysely_migration_lock', 'nip46_tokens', 'pubkey_domains', - 'unattached_media', 'nostr_events', - 'nostr_tags', - 'nostr_pgfts', 'event_zaps', ] ) { - await db.kysely.schema.dropTable(table).ifExists().cascade().execute(); + await kysely.schema.dropTable(table).ifExists().cascade().execute(); } - await db.kysely.destroy(); + await kysely.destroy(); } }, }; diff --git a/src/trends.ts b/src/trends.ts index 91164143..6199e9ed 100644 --- a/src/trends.ts +++ b/src/trends.ts @@ -1,9 +1,10 @@ import { NostrFilter } from '@nostrify/nostrify'; import { Stickynotes } from '@soapbox/stickynotes'; -import { sql } from 'kysely'; +import { Kysely, sql } from 'kysely'; import { Conf } from '@/config.ts'; -import { DittoDatabase, DittoDB } from '@/db/DittoDB.ts'; +import { DittoDB } from '@/db/DittoDB.ts'; +import { DittoTables } from '@/db/DittoTables.ts'; import { handleEvent } from '@/pipeline.ts'; import { AdminSigner } from '@/signers/AdminSigner.ts'; import { Time } from '@/utils/time.ts'; @@ -13,88 +14,50 @@ const console = new Stickynotes('ditto:trends'); /** Get trending tag values for a given tag in the given time frame. */ export async function getTrendingTagValues( /** Kysely instance to execute queries on. */ - { dialect, kysely }: DittoDatabase, + kysely: Kysely, /** Tag name to filter by, eg `t` or `r`. */ tagNames: string[], /** Filter of eligible events. */ filter: NostrFilter, ): Promise<{ value: string; authors: number; uses: number }[]> { - if (dialect === 'postgres') { - let query = kysely - .selectFrom([ - 'nostr_events', - sql<{ key: string; value: string }>`jsonb_each_text(nostr_events.tags_index)`.as('kv'), - sql<{ key: string; value: string }>`jsonb_array_elements_text(kv.value::jsonb)`.as('element'), - ]) - .select(({ fn }) => [ - fn('lower', ['element.value']).as('value'), - fn.agg('count', ['nostr_events.pubkey']).distinct().as('authors'), - fn.countAll().as('uses'), - ]) - .where('kv.key', '=', (eb) => eb.fn.any(eb.val(tagNames))) - .groupBy((eb) => eb.fn('lower', ['element.value'])) - .orderBy((eb) => eb.fn.agg('count', ['nostr_events.pubkey']).distinct(), 'desc'); + let query = kysely + .selectFrom([ + 'nostr_events', + sql<{ key: string; value: string }>`jsonb_each_text(nostr_events.tags_index)`.as('kv'), + sql<{ key: string; value: string }>`jsonb_array_elements_text(kv.value::jsonb)`.as('element'), + ]) + .select(({ fn }) => [ + fn('lower', ['element.value']).as('value'), + fn.agg('count', ['nostr_events.pubkey']).distinct().as('authors'), + fn.countAll().as('uses'), + ]) + .where('kv.key', '=', (eb) => eb.fn.any(eb.val(tagNames))) + .groupBy((eb) => eb.fn('lower', ['element.value'])) + .orderBy((eb) => eb.fn.agg('count', ['nostr_events.pubkey']).distinct(), 'desc'); - if (filter.kinds) { - query = query.where('nostr_events.kind', '=', ({ fn, val }) => fn.any(val(filter.kinds))); - } - if (filter.authors) { - query = query.where('nostr_events.pubkey', '=', ({ fn, val }) => fn.any(val(filter.authors))); - } - if (typeof filter.since === 'number') { - query = query.where('nostr_events.created_at', '>=', filter.since); - } - if (typeof filter.until === 'number') { - query = query.where('nostr_events.created_at', '<=', filter.until); - } - if (typeof filter.limit === 'number') { - query = query.limit(filter.limit); - } - - const rows = await query.execute(); - - return rows.map((row) => ({ - value: row.value, - authors: Number(row.authors), - uses: Number(row.uses), - })); + if (filter.kinds) { + query = query.where('nostr_events.kind', '=', ({ fn, val }) => fn.any(val(filter.kinds))); + } + if (filter.authors) { + query = query.where('nostr_events.pubkey', '=', ({ fn, val }) => fn.any(val(filter.authors))); + } + if (typeof filter.since === 'number') { + query = query.where('nostr_events.created_at', '>=', filter.since); + } + if (typeof filter.until === 'number') { + query = query.where('nostr_events.created_at', '<=', filter.until); + } + if (typeof filter.limit === 'number') { + query = query.limit(filter.limit); } - if (dialect === 'sqlite') { - let query = kysely - .selectFrom('nostr_tags') - .select(({ fn }) => [ - 'nostr_tags.value', - fn.agg('count', ['nostr_tags.pubkey']).distinct().as('authors'), - fn.countAll().as('uses'), - ]) - .where('nostr_tags.name', 'in', tagNames) - .groupBy('nostr_tags.value') - .orderBy((c) => c.fn.agg('count', ['nostr_tags.pubkey']).distinct(), 'desc'); + const rows = await query.execute(); - if (filter.kinds) { - query = query.where('nostr_tags.kind', 'in', filter.kinds); - } - if (typeof filter.since === 'number') { - query = query.where('nostr_tags.created_at', '>=', filter.since); - } - if (typeof filter.until === 'number') { - query = query.where('nostr_tags.created_at', '<=', filter.until); - } - if (typeof filter.limit === 'number') { - query = query.limit(filter.limit); - } - - const rows = await query.execute(); - - return rows.map((row) => ({ - value: row.value, - authors: Number(row.authors), - uses: Number(row.uses), - })); - } - - return []; + return rows.map((row) => ({ + value: row.value, + authors: Number(row.authors), + uses: Number(row.uses), + })); } /** Get trending tags and publish an event with them. */ diff --git a/src/utils/SimpleLRU.ts b/src/utils/SimpleLRU.ts index f1bf6512..0f2b5b37 100644 --- a/src/utils/SimpleLRU.ts +++ b/src/utils/SimpleLRU.ts @@ -17,7 +17,7 @@ export class SimpleLRU< constructor(fetchFn: FetchFn, opts: LRUCache.Options) { this.cache = new LRUCache({ - fetchMethod: (key, _staleValue, { signal }) => fetchFn(key, { signal: signal as AbortSignal }), + fetchMethod: (key, _staleValue, { signal }) => fetchFn(key, { signal: signal as unknown as AbortSignal }), ...opts, }); } diff --git a/src/utils/stats.ts b/src/utils/stats.ts index ccba0a5b..e4d4d3f2 100644 --- a/src/utils/stats.ts +++ b/src/utils/stats.ts @@ -5,7 +5,6 @@ import { z } from 'zod'; import { DittoTables } from '@/db/DittoTables.ts'; import { findQuoteTag, findReplyTag, getTagSet } from '@/utils/tags.ts'; -import { Conf } from '@/config.ts'; interface UpdateStatsOpts { kysely: Kysely; @@ -197,16 +196,13 @@ export async function updateAuthorStats( notes_count: 0, }; - let query = kysely + const prev = await kysely .selectFrom('author_stats') .selectAll() - .where('pubkey', '=', pubkey); + .forUpdate() + .where('pubkey', '=', pubkey) + .executeTakeFirst(); - if (Conf.db.dialect === 'postgres') { - query = query.forUpdate(); - } - - const prev = await query.executeTakeFirst(); const stats = fn(prev ?? empty); if (prev) { @@ -249,16 +245,13 @@ export async function updateEventStats( reactions: '{}', }; - let query = kysely + const prev = await kysely .selectFrom('event_stats') .selectAll() - .where('event_id', '=', eventId); + .forUpdate() + .where('event_id', '=', eventId) + .executeTakeFirst(); - if (Conf.db.dialect === 'postgres') { - query = query.forUpdate(); - } - - const prev = await query.executeTakeFirst(); const stats = fn(prev ?? empty); if (prev) { diff --git a/src/workers/fetch.worker.ts b/src/workers/fetch.worker.ts index 0012088b..469c8ff0 100644 --- a/src/workers/fetch.worker.ts +++ b/src/workers/fetch.worker.ts @@ -1,7 +1,7 @@ import Debug from '@soapbox/stickynotes/debug'; import * as Comlink from 'comlink'; -import './handlers/abortsignal.ts'; +import '@/workers/handlers/abortsignal.ts'; import '@/sentry.ts'; const debug = Debug('ditto:fetch.worker'); diff --git a/src/workers/sqlite.ts b/src/workers/sqlite.ts deleted file mode 100644 index 154ec556..00000000 --- a/src/workers/sqlite.ts +++ /dev/null @@ -1,52 +0,0 @@ -import * as Comlink from 'comlink'; -import { asyncGeneratorTransferHandler } from 'comlink-async-generator'; -import { CompiledQuery, QueryResult } from 'kysely'; - -import type { SqliteWorker as _SqliteWorker } from './sqlite.worker.ts'; - -Comlink.transferHandlers.set('asyncGenerator', asyncGeneratorTransferHandler); - -class SqliteWorker { - #worker: Worker; - #client: ReturnType>; - #ready: Promise; - - constructor() { - this.#worker = new Worker(new URL('./sqlite.worker.ts', import.meta.url).href, { type: 'module' }); - this.#client = Comlink.wrap(this.#worker); - - this.#ready = new Promise((resolve) => { - const handleEvent = (event: MessageEvent) => { - if (event.data[0] === 'ready') { - this.#worker.removeEventListener('message', handleEvent); - resolve(); - } - }; - this.#worker.addEventListener('message', handleEvent); - }); - } - - async open(path: string): Promise { - await this.#ready; - return this.#client.open(path); - } - - async executeQuery(query: CompiledQuery): Promise> { - await this.#ready; - return this.#client.executeQuery(query) as Promise>; - } - - async *streamQuery(query: CompiledQuery): AsyncIterableIterator> { - await this.#ready; - - for await (const result of await this.#client.streamQuery(query)) { - yield result as QueryResult; - } - } - - destroy(): Promise { - return this.#client.destroy(); - } -} - -export default SqliteWorker; diff --git a/src/workers/sqlite.worker.ts b/src/workers/sqlite.worker.ts deleted file mode 100644 index 23839dbd..00000000 --- a/src/workers/sqlite.worker.ts +++ /dev/null @@ -1,42 +0,0 @@ -/// -import { Database as SQLite } from '@db/sqlite'; -import * as Comlink from 'comlink'; -import { CompiledQuery, QueryResult } from 'kysely'; -import { asyncGeneratorTransferHandler } from 'comlink-async-generator'; - -import '@/sentry.ts'; - -let db: SQLite | undefined; - -export const SqliteWorker = { - open(path: string): void { - db = new SQLite(path); - }, - executeQuery({ sql, parameters }: CompiledQuery): QueryResult { - if (!db) throw new Error('Database not open'); - - return { - rows: db!.prepare(sql).all(...parameters as any[]) as R[], - numAffectedRows: BigInt(db!.changes), - insertId: BigInt(db!.lastInsertRowId), - }; - }, - async *streamQuery({ sql, parameters }: CompiledQuery): AsyncIterableIterator> { - if (!db) throw new Error('Database not open'); - - const stmt = db.prepare(sql).bind(...parameters as any[]); - for (const row of stmt) { - yield { - rows: [row], - }; - } - }, - destroy() { - db?.close(); - }, -}; - -Comlink.transferHandlers.set('asyncGenerator', asyncGeneratorTransferHandler); -Comlink.expose(SqliteWorker); - -self.postMessage(['ready']); From f3ae200833ef4910be6f553c797db77a11c22498 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Wed, 11 Sep 2024 11:48:31 -0500 Subject: [PATCH 34/82] Simplify database interfaces, make tests use pglite --- .gitlab-ci.yml | 4 +- scripts/admin-event.ts | 4 +- scripts/admin-role.ts | 4 +- scripts/db-migrate.ts | 7 --- scripts/nostr-pull.ts | 4 +- scripts/stats-recompute.ts | 2 +- src/controllers/api/oauth.ts | 2 +- src/controllers/api/statuses.ts | 2 +- src/controllers/api/streaming.ts | 2 +- src/controllers/metrics.ts | 6 ++- src/db/DittoDB.ts | 48 +++++------------ src/db/DittoDatabase.ts | 13 +++++ src/db/adapters/DittoPglite.ts | 35 +++++------- src/db/adapters/DittoPostgres.ts | 68 +++++++++++------------- src/db/migrations/020_pgfts.ts | 18 +++---- src/db/migrations/021_pgfts_index.ts | 22 +++----- src/db/migrations/030_pg_events_jsonb.ts | 4 -- src/middleware/signerMiddleware.ts | 2 +- src/pipeline.test.ts | 10 ++-- src/pipeline.ts | 8 +-- src/storages.ts | 4 +- src/storages/hydrate.ts | 2 +- src/test.ts | 33 +----------- src/trends.ts | 4 +- src/workers/fetch.worker.ts | 2 + 25 files changed, 117 insertions(+), 193 deletions(-) create mode 100644 src/db/DittoDatabase.ts diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a64b111d..48c5b253 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -35,10 +35,10 @@ test: postgres: stage: test - script: deno task db:migrate && deno task test + script: sleep 1 && deno task test services: - postgres:16 variables: DITTO_NSEC: nsec1zyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zyg3zygs4rm7hz - DATABASE_URL: postgres://postgres:postgres@postgres:5432/postgres + TEST_DATABASE_URL: postgres://postgres:postgres@postgres:5432/postgres POSTGRES_HOST_AUTH_METHOD: trust diff --git a/scripts/admin-event.ts b/scripts/admin-event.ts index 71c957f4..313aa051 100644 --- a/scripts/admin-event.ts +++ b/scripts/admin-event.ts @@ -9,8 +9,8 @@ import { nostrNow } from '@/utils.ts'; const signer = new AdminSigner(); -const db = await DittoDB.getInstance(); -const eventsDB = new EventsDB(db); +const { kysely } = await DittoDB.getInstance(); +const eventsDB = new EventsDB(kysely); const readable = Deno.stdin.readable .pipeThrough(new TextDecoderStream()) diff --git a/scripts/admin-role.ts b/scripts/admin-role.ts index 3f3c53f2..99986817 100644 --- a/scripts/admin-role.ts +++ b/scripts/admin-role.ts @@ -6,8 +6,8 @@ import { AdminSigner } from '@/signers/AdminSigner.ts'; import { EventsDB } from '@/storages/EventsDB.ts'; import { nostrNow } from '@/utils.ts'; -const db = await DittoDB.getInstance(); -const eventsDB = new EventsDB(db); +const { kysely } = await DittoDB.getInstance(); +const eventsDB = new EventsDB(kysely); const [pubkeyOrNpub, role] = Deno.args; const pubkey = pubkeyOrNpub.startsWith('npub1') ? nip19.decode(pubkeyOrNpub as `npub1${string}`).data : pubkeyOrNpub; diff --git a/scripts/db-migrate.ts b/scripts/db-migrate.ts index ab0b9747..0e1d694d 100644 --- a/scripts/db-migrate.ts +++ b/scripts/db-migrate.ts @@ -1,11 +1,4 @@ -import { Conf } from '@/config.ts'; import { DittoDB } from '@/db/DittoDB.ts'; -import { sleep } from '@/test.ts'; - -if (Deno.env.get('CI') && Conf.db.dialect === 'postgres') { - console.info('Waiting 1 second for postgres to start...'); - await sleep(1_000); -} // This migrates kysely internally. const { kysely } = await DittoDB.getInstance(); diff --git a/scripts/nostr-pull.ts b/scripts/nostr-pull.ts index 0556b64a..4b9d51db 100644 --- a/scripts/nostr-pull.ts +++ b/scripts/nostr-pull.ts @@ -9,8 +9,8 @@ import { nip19 } from 'nostr-tools'; import { DittoDB } from '@/db/DittoDB.ts'; import { EventsDB } from '@/storages/EventsDB.ts'; -const db = await DittoDB.getInstance(); -const eventsDB = new EventsDB(db); +const { kysely } = await DittoDB.getInstance(); +const eventsDB = new EventsDB(kysely); interface ImportEventsOpts { profilesOnly: boolean; diff --git a/scripts/stats-recompute.ts b/scripts/stats-recompute.ts index 107a3167..7d6f721f 100644 --- a/scripts/stats-recompute.ts +++ b/scripts/stats-recompute.ts @@ -18,6 +18,6 @@ try { } const store = await Storages.db(); -const kysely = await DittoDB.getInstance(); +const { kysely } = await DittoDB.getInstance(); await refreshAuthorStats({ pubkey, kysely, store }); diff --git a/src/controllers/api/oauth.ts b/src/controllers/api/oauth.ts index 01f80bf1..a736f5ca 100644 --- a/src/controllers/api/oauth.ts +++ b/src/controllers/api/oauth.ts @@ -82,7 +82,7 @@ const createTokenController: AppController = async (c) => { async function getToken( { pubkey, secret, relays = [] }: { pubkey: string; secret?: string; relays?: string[] }, ): Promise<`token1${string}`> { - const kysely = await DittoDB.getInstance(); + const { kysely } = await DittoDB.getInstance(); const token = generateToken(); const serverSeckey = generateSecretKey(); diff --git a/src/controllers/api/statuses.ts b/src/controllers/api/statuses.ts index 4236db52..05b5022c 100644 --- a/src/controllers/api/statuses.ts +++ b/src/controllers/api/statuses.ts @@ -578,7 +578,7 @@ const zappedByController: AppController = async (c) => { const id = c.req.param('id'); const params = c.get('listPagination'); const store = await Storages.db(); - const kysely = await DittoDB.getInstance(); + const { kysely } = await DittoDB.getInstance(); const zaps = await kysely.selectFrom('event_zaps') .selectAll() diff --git a/src/controllers/api/streaming.ts b/src/controllers/api/streaming.ts index f70a6b06..047aa573 100644 --- a/src/controllers/api/streaming.ts +++ b/src/controllers/api/streaming.ts @@ -222,7 +222,7 @@ async function topicToFilter( async function getTokenPubkey(token: string): Promise { if (token.startsWith('token1')) { - const kysely = await DittoDB.getInstance(); + const { kysely } = await DittoDB.getInstance(); const { user_pubkey } = await kysely .selectFrom('nip46_tokens') diff --git a/src/controllers/metrics.ts b/src/controllers/metrics.ts index e85294c0..e25522ff 100644 --- a/src/controllers/metrics.ts +++ b/src/controllers/metrics.ts @@ -6,9 +6,11 @@ import { dbAvailableConnectionsGauge, dbPoolSizeGauge } from '@/metrics.ts'; /** Prometheus/OpenMetrics controller. */ export const metricsController: AppController = async (c) => { + const db = await DittoDB.getInstance(); + // Update some metrics at request time. - dbPoolSizeGauge.set(DittoDB.poolSize); - dbAvailableConnectionsGauge.set(DittoDB.availableConnections); + dbPoolSizeGauge.set(db.poolSize); + dbAvailableConnectionsGauge.set(db.availableConnections); const metrics = await register.metrics(); diff --git a/src/db/DittoDB.ts b/src/db/DittoDB.ts index 03638a98..63d2bfb1 100644 --- a/src/db/DittoDB.ts +++ b/src/db/DittoDB.ts @@ -6,57 +6,35 @@ import { FileMigrationProvider, Kysely, Migrator } from 'kysely'; import { Conf } from '@/config.ts'; import { DittoPglite } from '@/db/adapters/DittoPglite.ts'; import { DittoPostgres } from '@/db/adapters/DittoPostgres.ts'; +import { DittoDatabase, DittoDatabaseOpts } from '@/db/DittoDatabase.ts'; import { DittoTables } from '@/db/DittoTables.ts'; export class DittoDB { - private static kysely: Promise> | undefined; + private static db: DittoDatabase | undefined; - static getInstance(): Promise> { - if (!this.kysely) { - this.kysely = this._getInstance(); + /** Create (and migrate) the database if it isn't been already, or return the existing connection. */ + static async getInstance(): Promise { + if (!this.db) { + this.db = this.create(Conf.databaseUrl, { poolSize: Conf.pg.poolSize }); + await this.migrate(this.db.kysely); } - return this.kysely; + return this.db; } - static async _getInstance(): Promise> { - const { protocol } = new URL(Conf.databaseUrl); - - let kysely: Kysely; + /** Open a new database connection. */ + static create(databaseUrl: string, opts?: DittoDatabaseOpts): DittoDatabase { + const { protocol } = new URL(databaseUrl); switch (protocol) { case 'file:': case 'memory:': - kysely = await DittoPglite.getInstance(); - break; + return DittoPglite.create(databaseUrl); case 'postgres:': case 'postgresql:': - kysely = await DittoPostgres.getInstance(); - break; + return DittoPostgres.create(databaseUrl, opts); default: throw new Error('Unsupported database URL.'); } - - await this.migrate(kysely); - - return kysely; - } - - static get poolSize(): number { - const { protocol } = new URL(Conf.databaseUrl); - - if (['postgres:', 'postgresql:'].includes(protocol)) { - return DittoPostgres.poolSize; - } - return 1; - } - - static get availableConnections(): number { - const { protocol } = new URL(Conf.databaseUrl); - - if (['postgres:', 'postgresql:'].includes(protocol)) { - return DittoPostgres.availableConnections; - } - return 1; } /** Migrate the database to the latest version. */ diff --git a/src/db/DittoDatabase.ts b/src/db/DittoDatabase.ts new file mode 100644 index 00000000..530d9391 --- /dev/null +++ b/src/db/DittoDatabase.ts @@ -0,0 +1,13 @@ +import { Kysely } from 'kysely'; + +import { DittoTables } from '@/db/DittoTables.ts'; + +export interface DittoDatabase { + readonly kysely: Kysely; + readonly poolSize: number; + readonly availableConnections: number; +} + +export interface DittoDatabaseOpts { + poolSize?: number; +} diff --git a/src/db/adapters/DittoPglite.ts b/src/db/adapters/DittoPglite.ts index 80ea6087..4ec7d8a5 100644 --- a/src/db/adapters/DittoPglite.ts +++ b/src/db/adapters/DittoPglite.ts @@ -2,32 +2,23 @@ import { PGlite } from '@electric-sql/pglite'; import { PgliteDialect } from '@soapbox/kysely-pglite'; import { Kysely } from 'kysely'; -import { Conf } from '@/config.ts'; +import { DittoDatabase } from '@/db/DittoDatabase.ts'; import { DittoTables } from '@/db/DittoTables.ts'; import { KyselyLogger } from '@/db/KyselyLogger.ts'; export class DittoPglite { - static db: Kysely | undefined; + static create(databaseUrl: string): DittoDatabase { + const kysely = new Kysely({ + dialect: new PgliteDialect({ + database: new PGlite(databaseUrl), + }), + log: KyselyLogger, + }); - // deno-lint-ignore require-await - static async getInstance(): Promise> { - if (!this.db) { - this.db = new Kysely({ - dialect: new PgliteDialect({ - database: new PGlite(Conf.databaseUrl), - }), - log: KyselyLogger, - }) as Kysely; - } - - return this.db; - } - - static get poolSize() { - return 1; - } - - static get availableConnections(): number { - return 1; + return { + kysely, + poolSize: 1, + availableConnections: 1, + }; } } diff --git a/src/db/adapters/DittoPostgres.ts b/src/db/adapters/DittoPostgres.ts index b0a6c93a..f1a5bcc9 100644 --- a/src/db/adapters/DittoPostgres.ts +++ b/src/db/adapters/DittoPostgres.ts @@ -12,51 +12,43 @@ import { import { PostgresJSDialectConfig, PostgresJSDriver } from 'kysely-postgres-js'; import postgres from 'postgres'; -import { Conf } from '@/config.ts'; +import { DittoDatabase, DittoDatabaseOpts } from '@/db/DittoDatabase.ts'; import { DittoTables } from '@/db/DittoTables.ts'; import { KyselyLogger } from '@/db/KyselyLogger.ts'; export class DittoPostgres { - static kysely: Kysely | undefined; - static postgres?: postgres.Sql; + static create(databaseUrl: string, opts?: DittoDatabaseOpts): DittoDatabase { + const pg = postgres(databaseUrl, { max: opts?.poolSize }); - // deno-lint-ignore require-await - static async getInstance(): Promise> { - if (!this.postgres) { - this.postgres = postgres(Conf.databaseUrl, { max: Conf.pg.poolSize }); - } - - if (!this.kysely) { - this.kysely = new Kysely({ - dialect: { - createAdapter() { - return new PostgresAdapter(); - }, - createDriver() { - return new PostgresJSDriver({ - postgres: DittoPostgres.postgres as unknown as PostgresJSDialectConfig['postgres'], - }); - }, - createIntrospector(db) { - return new PostgresIntrospector(db); - }, - createQueryCompiler() { - return new DittoPostgresQueryCompiler(); - }, + const kysely = new Kysely({ + dialect: { + createAdapter() { + return new PostgresAdapter(); }, - log: KyselyLogger, - }); - } + createDriver() { + return new PostgresJSDriver({ + postgres: pg as unknown as PostgresJSDialectConfig['postgres'], + }); + }, + createIntrospector(db) { + return new PostgresIntrospector(db); + }, + createQueryCompiler() { + return new DittoPostgresQueryCompiler(); + }, + }, + log: KyselyLogger, + }); - return this.kysely; - } - - static get poolSize() { - return this.postgres?.connections.open ?? 0; - } - - static get availableConnections(): number { - return this.postgres?.connections.idle ?? 0; + return { + kysely, + get poolSize() { + return pg.connections.open; + }, + get availableConnections() { + return pg.connections.idle; + }, + }; } } diff --git a/src/db/migrations/020_pgfts.ts b/src/db/migrations/020_pgfts.ts index 835de117..26e320ec 100644 --- a/src/db/migrations/020_pgfts.ts +++ b/src/db/migrations/020_pgfts.ts @@ -1,19 +1,13 @@ import { Kysely, sql } from 'kysely'; -import { Conf } from '@/config.ts'; - export async function up(db: Kysely): Promise { - if (Conf.db.dialect === 'postgres') { - await db.schema.createTable('nostr_pgfts') - .ifNotExists() - .addColumn('event_id', 'text', (c) => c.primaryKey().references('nostr_events.id').onDelete('cascade')) - .addColumn('search_vec', sql`tsvector`, (c) => c.notNull()) - .execute(); - } + await db.schema.createTable('nostr_pgfts') + .ifNotExists() + .addColumn('event_id', 'text', (c) => c.primaryKey().references('nostr_events.id').onDelete('cascade')) + .addColumn('search_vec', sql`tsvector`, (c) => c.notNull()) + .execute(); } export async function down(db: Kysely): Promise { - if (Conf.db.dialect === 'postgres') { - await db.schema.dropTable('nostr_pgfts').ifExists().execute(); - } + await db.schema.dropTable('nostr_pgfts').ifExists().execute(); } diff --git a/src/db/migrations/021_pgfts_index.ts b/src/db/migrations/021_pgfts_index.ts index 4b834995..7ad24546 100644 --- a/src/db/migrations/021_pgfts_index.ts +++ b/src/db/migrations/021_pgfts_index.ts @@ -1,21 +1,15 @@ import { Kysely } from 'kysely'; -import { Conf } from '@/config.ts'; - export async function up(db: Kysely): Promise { - if (Conf.db.dialect === 'postgres') { - await db.schema - .createIndex('nostr_pgfts_gin_search_vec') - .ifNotExists() - .on('nostr_pgfts') - .using('gin') - .column('search_vec') - .execute(); - } + await db.schema + .createIndex('nostr_pgfts_gin_search_vec') + .ifNotExists() + .on('nostr_pgfts') + .using('gin') + .column('search_vec') + .execute(); } export async function down(db: Kysely): Promise { - if (Conf.db.dialect === 'postgres') { - await db.schema.dropIndex('nostr_pgfts_gin_search_vec').ifExists().execute(); - } + await db.schema.dropIndex('nostr_pgfts_gin_search_vec').ifExists().execute(); } diff --git a/src/db/migrations/030_pg_events_jsonb.ts b/src/db/migrations/030_pg_events_jsonb.ts index 7bfc6c17..dcd6ad85 100644 --- a/src/db/migrations/030_pg_events_jsonb.ts +++ b/src/db/migrations/030_pg_events_jsonb.ts @@ -1,10 +1,6 @@ import { Kysely, sql } from 'kysely'; -import { Conf } from '@/config.ts'; - export async function up(db: Kysely): Promise { - if (Conf.db.dialect !== 'postgres') return; - // Create new table and indexes. await db.schema .createTable('nostr_events_new') diff --git a/src/middleware/signerMiddleware.ts b/src/middleware/signerMiddleware.ts index 89a494c4..60826db9 100644 --- a/src/middleware/signerMiddleware.ts +++ b/src/middleware/signerMiddleware.ts @@ -20,7 +20,7 @@ export const signerMiddleware: AppMiddleware = async (c, next) => { if (bech32.startsWith('token1')) { try { - const kysely = await DittoDB.getInstance(); + const { kysely } = await DittoDB.getInstance(); const { user_pubkey, server_seckey, relays } = await kysely .selectFrom('nip46_tokens') diff --git a/src/pipeline.test.ts b/src/pipeline.test.ts index 2af2b8c3..76b1fe51 100644 --- a/src/pipeline.test.ts +++ b/src/pipeline.test.ts @@ -1,7 +1,7 @@ import { assertEquals } from '@std/assert'; import { generateSecretKey } from 'nostr-tools'; -import { createTestDB, genEvent, getTestDB } from '@/test.ts'; +import { createTestDB, genEvent } from '@/test.ts'; import { handleZaps } from '@/pipeline.ts'; Deno.test('store one zap receipt in nostr_events; convert it into event_zaps table format and store it', async () => { @@ -58,7 +58,7 @@ Deno.test('store one zap receipt in nostr_events; convert it into event_zaps tab // If no error happens = ok Deno.test('zap receipt does not have a "description" tag', async () => { - await using db = await getTestDB(); + await using db = await createTestDB(); const kysely = db.kysely; const sk = generateSecretKey(); @@ -71,7 +71,7 @@ Deno.test('zap receipt does not have a "description" tag', async () => { }); Deno.test('zap receipt does not have a zap request stringified value in the "description" tag', async () => { - await using db = await getTestDB(); + await using db = await createTestDB(); const kysely = db.kysely; const sk = generateSecretKey(); @@ -84,7 +84,7 @@ Deno.test('zap receipt does not have a zap request stringified value in the "des }); Deno.test('zap receipt does not have a "bolt11" tag', async () => { - await using db = await getTestDB(); + await using db = await createTestDB(); const kysely = db.kysely; const sk = generateSecretKey(); @@ -103,7 +103,7 @@ Deno.test('zap receipt does not have a "bolt11" tag', async () => { }); Deno.test('zap request inside zap receipt does not have an "e" tag', async () => { - await using db = await getTestDB(); + await using db = await createTestDB(); const kysely = db.kysely; const sk = generateSecretKey(); diff --git a/src/pipeline.ts b/src/pipeline.ts index ceb370ca..dd59cb8d 100644 --- a/src/pipeline.ts +++ b/src/pipeline.ts @@ -53,7 +53,7 @@ async function handleEvent(event: DittoEvent, signal: AbortSignal): Promise { async function hydrateEvent(event: DittoEvent, signal: AbortSignal): Promise { await hydrateEvents({ events: [event], store: await Storages.db(), signal }); - const kysely = await DittoDB.getInstance(); + const { kysely } = await DittoDB.getInstance(); const domain = await kysely .selectFrom('pubkey_domains') .select('domain') @@ -118,7 +118,7 @@ async function hydrateEvent(event: DittoEvent, signal: AbortSignal): Promise { if (NKinds.ephemeral(event.kind)) return; const store = await Storages.db(); - const kysely = await DittoDB.getInstance(); + const { kysely } = await DittoDB.getInstance(); await updateStats({ event, store, kysely }).catch(debug); await store.event(event, { signal }); @@ -146,7 +146,7 @@ async function parseMetadata(event: NostrEvent, signal: AbortSignal): Promise { if (!this._db) { this._db = (async () => { - const db = await DittoDB.getInstance(); - const store = new EventsDB(db); + const { kysely } = await DittoDB.getInstance(); + const store = new EventsDB(kysely); await seedZapSplits(store); return store; })(); diff --git a/src/storages/hydrate.ts b/src/storages/hydrate.ts index 2294f880..19ba0db4 100644 --- a/src/storages/hydrate.ts +++ b/src/storages/hydrate.ts @@ -18,7 +18,7 @@ interface HydrateOpts { /** Hydrate events using the provided storage. */ async function hydrateEvents(opts: HydrateOpts): Promise { - const { events, store, signal, kysely = await DittoDB.getInstance() } = opts; + const { events, store, signal, kysely = (await DittoDB.getInstance()).kysely } = opts; if (!events.length) { return events; diff --git a/src/test.ts b/src/test.ts index 3f144eb6..444a1b1c 100644 --- a/src/test.ts +++ b/src/test.ts @@ -1,16 +1,9 @@ -import { PGlite } from '@electric-sql/pglite'; import { NostrEvent } from '@nostrify/nostrify'; -import { PgliteDialect } from '@soapbox/kysely-pglite'; import { finalizeEvent, generateSecretKey } from 'nostr-tools'; -import { Kysely } from 'kysely'; -import { PostgresJSDialect, PostgresJSDialectConfig } from 'kysely-postgres-js'; -import postgres from 'postgres'; import { Conf } from '@/config.ts'; import { DittoDB } from '@/db/DittoDB.ts'; -import { DittoTables } from '@/db/DittoTables.ts'; import { purifyEvent } from '@/storages/hydrate.ts'; -import { KyselyLogger } from '@/db/KyselyLogger.ts'; import { EventsDB } from '@/storages/EventsDB.ts'; /** Import an event fixture by name in tests. */ @@ -41,31 +34,7 @@ export function genEvent(t: Partial = {}, sk: Uint8Array = generateS /** Create an database for testing. */ export const createTestDB = async (databaseUrl = Conf.testDatabaseUrl) => { const { protocol } = new URL(databaseUrl); - - const kysely: Kysely = (() => { - switch (protocol) { - case 'postgres:': - case 'postgresql:': - return new Kysely({ - // @ts-ignore Kysely version mismatch. - dialect: new PostgresJSDialect({ - postgres: postgres(databaseUrl, { - max: Conf.pg.poolSize, - }) as unknown as PostgresJSDialectConfig['postgres'], - }), - log: KyselyLogger, - }); - case 'file:': - case 'memory:': - return new Kysely({ - dialect: new PgliteDialect({ - database: new PGlite(databaseUrl), - }), - }); - default: - throw new Error(`Unsupported database URL protocol: ${protocol}`); - } - })(); + const { kysely } = DittoDB.create(databaseUrl, { poolSize: 1 }); await DittoDB.migrate(kysely); const store = new EventsDB(kysely); diff --git a/src/trends.ts b/src/trends.ts index 6199e9ed..337ee5c7 100644 --- a/src/trends.ts +++ b/src/trends.ts @@ -70,7 +70,7 @@ export async function updateTrendingTags( aliases?: string[], ) { console.info(`Updating trending ${l}...`); - const db = await DittoDB.getInstance(); + const { kysely } = await DittoDB.getInstance(); const signal = AbortSignal.timeout(1000); const yesterday = Math.floor((Date.now() - Time.days(1)) / 1000); @@ -79,7 +79,7 @@ export async function updateTrendingTags( const tagNames = aliases ? [tagName, ...aliases] : [tagName]; try { - const trends = await getTrendingTagValues(db, tagNames, { + const trends = await getTrendingTagValues(kysely, tagNames, { kinds, since: yesterday, until: now, diff --git a/src/workers/fetch.worker.ts b/src/workers/fetch.worker.ts index 469c8ff0..e6f98455 100644 --- a/src/workers/fetch.worker.ts +++ b/src/workers/fetch.worker.ts @@ -1,3 +1,5 @@ +/// + import Debug from '@soapbox/stickynotes/debug'; import * as Comlink from 'comlink'; From fad40f1c30d61082dd717105336d413ebe56f6aa Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Wed, 11 Sep 2024 11:57:28 -0500 Subject: [PATCH 35/82] createTestDB: don't accept an arg --- src/test.ts | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/test.ts b/src/test.ts index 444a1b1c..8b3dad80 100644 --- a/src/test.ts +++ b/src/test.ts @@ -31,10 +31,11 @@ export function genEvent(t: Partial = {}, sk: Uint8Array = generateS return purifyEvent(event); } -/** Create an database for testing. */ -export const createTestDB = async (databaseUrl = Conf.testDatabaseUrl) => { - const { protocol } = new URL(databaseUrl); - const { kysely } = DittoDB.create(databaseUrl, { poolSize: 1 }); +/** Create a database for testing. It uses `TEST_DATABASE_URL`, or creates an in-memory database by default. */ +export async function createTestDB() { + const { testDatabaseUrl } = Conf; + const { protocol } = new URL(testDatabaseUrl); + const { kysely } = DittoDB.create(testDatabaseUrl, { poolSize: 1 }); await DittoDB.migrate(kysely); const store = new EventsDB(kysely); @@ -65,7 +66,7 @@ export const createTestDB = async (databaseUrl = Conf.testDatabaseUrl) => { } }, }; -}; +} export function sleep(ms: number): Promise { return new Promise((resolve) => setTimeout(resolve, ms)); From d2fb3fd2534d3c722bc6533c54fad0261157e29d Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Wed, 11 Sep 2024 13:06:20 -0500 Subject: [PATCH 36/82] Make EventsDB not rely on Conf --- deno.json | 1 - scripts/admin-event.ts | 9 +++----- scripts/admin-role.ts | 10 ++++----- scripts/nostr-pull.ts | 8 +++---- src/controllers/api/statuses.ts | 3 ++- src/storages.ts | 2 +- src/storages/EventsDB.ts | 37 ++++++++++++++++++++------------- src/storages/InternalRelay.ts | 2 +- src/storages/hydrate.ts | 17 ++------------- src/test.ts | 9 ++++++-- src/utils/api.ts | 2 +- src/utils/purify.ts | 14 +++++++++++++ src/workers/policy.ts | 2 +- src/workers/policy.worker.ts | 15 +++++++++++-- 14 files changed, 75 insertions(+), 56 deletions(-) create mode 100644 src/utils/purify.ts diff --git a/deno.json b/deno.json index 4897cff4..699ab620 100644 --- a/deno.json +++ b/deno.json @@ -1,5 +1,4 @@ { - "$schema": "https://deno.land/x/deno@v1.41.0/cli/schemas/config-file.v1.json", "version": "1.1.0", "tasks": { "start": "deno run -A src/server.ts", diff --git a/scripts/admin-event.ts b/scripts/admin-event.ts index 313aa051..00711993 100644 --- a/scripts/admin-event.ts +++ b/scripts/admin-event.ts @@ -1,16 +1,13 @@ import { JsonParseStream } from '@std/json/json-parse-stream'; import { TextLineStream } from '@std/streams/text-line-stream'; -import { DittoDB } from '@/db/DittoDB.ts'; import { AdminSigner } from '@/signers/AdminSigner.ts'; -import { EventsDB } from '@/storages/EventsDB.ts'; +import { Storages } from '@/storages.ts'; import { type EventStub } from '@/utils/api.ts'; import { nostrNow } from '@/utils.ts'; const signer = new AdminSigner(); - -const { kysely } = await DittoDB.getInstance(); -const eventsDB = new EventsDB(kysely); +const store = await Storages.db(); const readable = Deno.stdin.readable .pipeThrough(new TextDecoderStream()) @@ -25,7 +22,7 @@ for await (const t of readable) { ...t as EventStub, }); - await eventsDB.event(event); + await store.event(event); } Deno.exit(0); diff --git a/scripts/admin-role.ts b/scripts/admin-role.ts index 99986817..d275329f 100644 --- a/scripts/admin-role.ts +++ b/scripts/admin-role.ts @@ -1,13 +1,11 @@ import { NSchema } from '@nostrify/nostrify'; import { nip19 } from 'nostr-tools'; -import { DittoDB } from '@/db/DittoDB.ts'; import { AdminSigner } from '@/signers/AdminSigner.ts'; -import { EventsDB } from '@/storages/EventsDB.ts'; +import { Storages } from '@/storages.ts'; import { nostrNow } from '@/utils.ts'; -const { kysely } = await DittoDB.getInstance(); -const eventsDB = new EventsDB(kysely); +const store = await Storages.db(); const [pubkeyOrNpub, role] = Deno.args; const pubkey = pubkeyOrNpub.startsWith('npub1') ? nip19.decode(pubkeyOrNpub as `npub1${string}`).data : pubkeyOrNpub; @@ -25,7 +23,7 @@ if (!['admin', 'user'].includes(role)) { const signer = new AdminSigner(); const admin = await signer.getPublicKey(); -const [existing] = await eventsDB.query([{ +const [existing] = await store.query([{ kinds: [30382], authors: [admin], '#d': [pubkey], @@ -59,6 +57,6 @@ const event = await signer.signEvent({ created_at: nostrNow(), }); -await eventsDB.event(event); +await store.event(event); Deno.exit(0); diff --git a/scripts/nostr-pull.ts b/scripts/nostr-pull.ts index 4b9d51db..f7a3840c 100644 --- a/scripts/nostr-pull.ts +++ b/scripts/nostr-pull.ts @@ -6,11 +6,9 @@ import { NostrEvent, NRelay1, NSchema } from '@nostrify/nostrify'; import { nip19 } from 'nostr-tools'; -import { DittoDB } from '@/db/DittoDB.ts'; -import { EventsDB } from '@/storages/EventsDB.ts'; +import { Storages } from '@/storages.ts'; -const { kysely } = await DittoDB.getInstance(); -const eventsDB = new EventsDB(kysely); +const store = await Storages.db(); interface ImportEventsOpts { profilesOnly: boolean; @@ -21,7 +19,7 @@ const importUsers = async ( authors: string[], relays: string[], opts?: Partial, - doEvent: DoEvent = async (event: NostrEvent) => await eventsDB.event(event), + doEvent: DoEvent = async (event: NostrEvent) => await store.event(event), ) => { // Kind 0s + follow lists. const profiles: Record> = {}; diff --git a/src/controllers/api/statuses.ts b/src/controllers/api/statuses.ts index 05b5022c..bef98122 100644 --- a/src/controllers/api/statuses.ts +++ b/src/controllers/api/statuses.ts @@ -16,9 +16,10 @@ import { addTag, deleteTag } from '@/utils/tags.ts'; import { asyncReplaceAll } from '@/utils/text.ts'; import { lookupPubkey } from '@/utils/lookup.ts'; import { Storages } from '@/storages.ts'; -import { hydrateEvents, purifyEvent } from '@/storages/hydrate.ts'; +import { hydrateEvents } from '@/storages/hydrate.ts'; import { createEvent, paginated, paginatedList, parseBody, updateListEvent } from '@/utils/api.ts'; import { getInvoice, getLnurl } from '@/utils/lnurl.ts'; +import { purifyEvent } from '@/utils/purify.ts'; import { getZapSplits } from '@/utils/zap-split.ts'; import { renderEventAccounts } from '@/views.ts'; import { accountFromPubkey, renderAccount } from '@/views/mastodon/accounts.ts'; diff --git a/src/storages.ts b/src/storages.ts index 7114a3d6..10510104 100644 --- a/src/storages.ts +++ b/src/storages.ts @@ -21,7 +21,7 @@ export class Storages { if (!this._db) { this._db = (async () => { const { kysely } = await DittoDB.getInstance(); - const store = new EventsDB(kysely); + const store = new EventsDB({ kysely, pubkey: Conf.pubkey, timeout: Conf.db.timeouts.default }); await seedZapSplits(store); return store; })(); diff --git a/src/storages/EventsDB.ts b/src/storages/EventsDB.ts index 6c006dc2..72cd9bb3 100644 --- a/src/storages/EventsDB.ts +++ b/src/storages/EventsDB.ts @@ -1,6 +1,6 @@ // deno-lint-ignore-file require-await -import { NDatabase, NPostgres } from '@nostrify/db'; +import { NPostgres } from '@nostrify/db'; import { NIP50, NKinds, @@ -16,13 +16,12 @@ import { Stickynotes } from '@soapbox/stickynotes'; import { Kysely } from 'kysely'; import { nip27 } from 'nostr-tools'; -import { Conf } from '@/config.ts'; import { DittoTables } from '@/db/DittoTables.ts'; import { dbEventsCounter } from '@/metrics.ts'; import { RelayError } from '@/RelayError.ts'; -import { purifyEvent } from '@/storages/hydrate.ts'; import { isNostrId, isURL } from '@/utils.ts'; import { abortError } from '@/utils/abort.ts'; +import { purifyEvent } from '@/utils/purify.ts'; /** Function to decide whether or not to index a tag. */ type TagCondition = ({ event, count, value }: { @@ -31,9 +30,19 @@ type TagCondition = ({ event, count, value }: { value: string; }) => boolean; +/** Options for the EventsDB store. */ +interface EventsDBOpts { + /** Kysely instance to use. */ + kysely: Kysely; + /** Pubkey of the admin account. */ + pubkey: string; + /** Timeout in milliseconds for database queries. */ + timeout: number; +} + /** SQL database storage adapter for Nostr events. */ class EventsDB implements NStore { - private store: NDatabase | NPostgres; + private store: NPostgres; private console = new Stickynotes('ditto:db:events'); /** Conditions for when to index certain tags. */ @@ -53,8 +62,8 @@ class EventsDB implements NStore { 't': ({ event, count, value }) => (event.kind === 1985 ? count < 20 : count < 5) && value.length < 50, }; - constructor(private kysely: Kysely) { - this.store = new NPostgres(kysely, { + constructor(private opts: EventsDBOpts) { + this.store = new NPostgres(opts.kysely, { indexTags: EventsDB.indexTags, indexSearch: EventsDB.searchText, }); @@ -73,7 +82,7 @@ class EventsDB implements NStore { await this.deleteEventsAdmin(event); try { - await this.store.event(event, { ...opts, timeout: opts.timeout ?? Conf.db.timeouts.default }); + await this.store.event(event, { ...opts, timeout: opts.timeout ?? this.opts.timeout }); } catch (e) { if (e.message === 'Cannot add a deleted event') { throw new RelayError('blocked', 'event deleted by user'); @@ -88,7 +97,7 @@ class EventsDB implements NStore { /** Check if an event has been deleted by the admin. */ private async isDeletedAdmin(event: NostrEvent): Promise { const filters: NostrFilter[] = [ - { kinds: [5], authors: [Conf.pubkey], '#e': [event.id], limit: 1 }, + { kinds: [5], authors: [this.opts.pubkey], '#e': [event.id], limit: 1 }, ]; if (NKinds.replaceable(event.kind) || NKinds.parameterizedReplaceable(event.kind)) { @@ -96,7 +105,7 @@ class EventsDB implements NStore { filters.push({ kinds: [5], - authors: [Conf.pubkey], + authors: [this.opts.pubkey], '#a': [`${event.kind}:${event.pubkey}:${d}`], since: event.created_at, limit: 1, @@ -109,7 +118,7 @@ class EventsDB implements NStore { /** The DITTO_NSEC can delete any event from the database. NDatabase already handles user deletions. */ private async deleteEventsAdmin(event: NostrEvent): Promise { - if (event.kind === 5 && event.pubkey === Conf.pubkey) { + if (event.kind === 5 && event.pubkey === this.opts.pubkey) { const ids = new Set(event.tags.filter(([name]) => name === 'e').map(([_name, value]) => value)); const addrs = new Set(event.tags.filter(([name]) => name === 'a').map(([_name, value]) => value)); @@ -180,7 +189,7 @@ class EventsDB implements NStore { this.console.debug('REQ', JSON.stringify(filters)); - return this.store.query(filters, { ...opts, timeout: opts.timeout ?? Conf.db.timeouts.default }); + return this.store.query(filters, { ...opts, timeout: opts.timeout ?? this.opts.timeout }); } /** Delete events based on filters from the database. */ @@ -188,7 +197,7 @@ class EventsDB implements NStore { if (!filters.length) return Promise.resolve(); this.console.debug('DELETE', JSON.stringify(filters)); - return this.store.remove(filters, { ...opts, timeout: opts.timeout ?? Conf.db.timeouts.default }); + return this.store.remove(filters, { ...opts, timeout: opts.timeout ?? this.opts.timeout }); } /** Get number of events that would be returned by filters. */ @@ -201,7 +210,7 @@ class EventsDB implements NStore { this.console.debug('COUNT', JSON.stringify(filters)); - return this.store.count(filters, { ...opts, timeout: opts.timeout ?? Conf.db.timeouts.default }); + return this.store.count(filters, { ...opts, timeout: opts.timeout ?? this.opts.timeout }); } /** Return only the tags that should be indexed. */ @@ -277,7 +286,7 @@ class EventsDB implements NStore { ) as { key: 'domain'; value: string } | undefined)?.value; if (domain) { - const query = this.kysely + const query = this.opts.kysely .selectFrom('pubkey_domains') .select('pubkey') .where('domain', '=', domain); diff --git a/src/storages/InternalRelay.ts b/src/storages/InternalRelay.ts index 233a095c..93a480e1 100644 --- a/src/storages/InternalRelay.ts +++ b/src/storages/InternalRelay.ts @@ -12,7 +12,7 @@ import { Machina } from '@nostrify/nostrify/utils'; import { matchFilter } from 'nostr-tools'; import { DittoEvent } from '@/interfaces/DittoEvent.ts'; -import { purifyEvent } from '@/storages/hydrate.ts'; +import { purifyEvent } from '@/utils/purify.ts'; /** * PubSub event store for streaming events within the application. diff --git a/src/storages/hydrate.ts b/src/storages/hydrate.ts index 19ba0db4..c7a277c7 100644 --- a/src/storages/hydrate.ts +++ b/src/storages/hydrate.ts @@ -1,4 +1,4 @@ -import { NostrEvent, NStore } from '@nostrify/nostrify'; +import { NStore } from '@nostrify/nostrify'; import { matchFilter } from 'nostr-tools'; import { DittoDB } from '@/db/DittoDB.ts'; @@ -338,17 +338,4 @@ async function gatherEventStats( })); } -/** Return a normalized event without any non-standard keys. */ -function purifyEvent(event: NostrEvent): NostrEvent { - return { - id: event.id, - pubkey: event.pubkey, - kind: event.kind, - content: event.content, - tags: event.tags, - sig: event.sig, - created_at: event.created_at, - }; -} - -export { hydrateEvents, purifyEvent }; +export { hydrateEvents }; diff --git a/src/test.ts b/src/test.ts index 8b3dad80..45946f00 100644 --- a/src/test.ts +++ b/src/test.ts @@ -3,8 +3,8 @@ import { finalizeEvent, generateSecretKey } from 'nostr-tools'; import { Conf } from '@/config.ts'; import { DittoDB } from '@/db/DittoDB.ts'; -import { purifyEvent } from '@/storages/hydrate.ts'; import { EventsDB } from '@/storages/EventsDB.ts'; +import { purifyEvent } from '@/utils/purify.ts'; /** Import an event fixture by name in tests. */ export async function eventFixture(name: string): Promise { @@ -38,7 +38,12 @@ export async function createTestDB() { const { kysely } = DittoDB.create(testDatabaseUrl, { poolSize: 1 }); await DittoDB.migrate(kysely); - const store = new EventsDB(kysely); + + const store = new EventsDB({ + kysely, + timeout: Conf.db.timeouts.default, + pubkey: Conf.pubkey, + }); return { store, diff --git a/src/utils/api.ts b/src/utils/api.ts index b3b5a8b1..c6d3c6b6 100644 --- a/src/utils/api.ts +++ b/src/utils/api.ts @@ -13,7 +13,7 @@ import { RelayError } from '@/RelayError.ts'; import { AdminSigner } from '@/signers/AdminSigner.ts'; import { Storages } from '@/storages.ts'; import { nostrNow } from '@/utils.ts'; -import { purifyEvent } from '@/storages/hydrate.ts'; +import { purifyEvent } from '@/utils/purify.ts'; const debug = Debug('ditto:api'); diff --git a/src/utils/purify.ts b/src/utils/purify.ts new file mode 100644 index 00000000..84c1e44b --- /dev/null +++ b/src/utils/purify.ts @@ -0,0 +1,14 @@ +import { NostrEvent } from '@nostrify/nostrify'; + +/** Return a normalized event without any non-standard keys. */ +export function purifyEvent(event: NostrEvent): NostrEvent { + return { + id: event.id, + pubkey: event.pubkey, + kind: event.kind, + content: event.content, + tags: event.tags, + sig: event.sig, + created_at: event.created_at, + }; +} diff --git a/src/workers/policy.ts b/src/workers/policy.ts index ef9aa2cd..08511ded 100644 --- a/src/workers/policy.ts +++ b/src/workers/policy.ts @@ -24,7 +24,7 @@ export const policyWorker = Comlink.wrap( ); try { - await policyWorker.import(Conf.policy); + await policyWorker.init(Conf.policy, Conf.databaseUrl, Conf.pubkey); console.debug(`Using custom policy: ${Conf.policy}`); } catch (e) { if (e.message.includes('Module not found')) { diff --git a/src/workers/policy.worker.ts b/src/workers/policy.worker.ts index 9f94a008..0036c4bd 100644 --- a/src/workers/policy.worker.ts +++ b/src/workers/policy.worker.ts @@ -3,6 +3,9 @@ import { NostrEvent, NostrRelayOK, NPolicy } from '@nostrify/nostrify'; import { NoOpPolicy, ReadOnlyPolicy } from '@nostrify/nostrify/policies'; import * as Comlink from 'comlink'; +import { DittoDB } from '@/db/DittoDB.ts'; +import { EventsDB } from '@/storages/EventsDB.ts'; + export class CustomPolicy implements NPolicy { private policy: NPolicy = new ReadOnlyPolicy(); @@ -11,10 +14,18 @@ export class CustomPolicy implements NPolicy { return this.policy.call(event); } - async import(path: string): Promise { + async init(path: string, databaseUrl: string, adminPubkey: string): Promise { + const { kysely } = DittoDB.create(databaseUrl, { poolSize: 1 }); + + const store = new EventsDB({ + kysely, + pubkey: adminPubkey, + timeout: 1_000, + }); + try { const Policy = (await import(path)).default; - this.policy = new Policy(); + this.policy = new Policy({ store }); } catch (e) { if (e.message.includes('Module not found')) { this.policy = new NoOpPolicy(); From ebc0250d81f74bd25e741e2d588f5b44a75af6f9 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Wed, 11 Sep 2024 13:23:06 -0500 Subject: [PATCH 37/82] DittoDB.getInstance() -> Storages.kysely() --- scripts/db-migrate.ts | 4 ++-- scripts/stats-recompute.ts | 3 +-- src/controllers/api/oauth.ts | 3 +-- src/controllers/api/statuses.ts | 3 +-- src/controllers/api/streaming.ts | 3 +-- src/controllers/metrics.ts | 4 ++-- src/db/DittoDB.ts | 12 ------------ src/middleware/signerMiddleware.ts | 4 ++-- src/pipeline.ts | 9 ++++----- src/storages.ts | 17 ++++++++++++++++- src/storages/hydrate.ts | 6 +++--- src/trends.ts | 4 ++-- 12 files changed, 35 insertions(+), 37 deletions(-) diff --git a/scripts/db-migrate.ts b/scripts/db-migrate.ts index 0e1d694d..d3e93783 100644 --- a/scripts/db-migrate.ts +++ b/scripts/db-migrate.ts @@ -1,7 +1,7 @@ -import { DittoDB } from '@/db/DittoDB.ts'; +import { Storages } from '@/storages.ts'; // This migrates kysely internally. -const { kysely } = await DittoDB.getInstance(); +const kysely = await Storages.kysely(); // Close the connection before exiting. await kysely.destroy(); diff --git a/scripts/stats-recompute.ts b/scripts/stats-recompute.ts index 7d6f721f..77be13fe 100644 --- a/scripts/stats-recompute.ts +++ b/scripts/stats-recompute.ts @@ -1,6 +1,5 @@ import { nip19 } from 'nostr-tools'; -import { DittoDB } from '@/db/DittoDB.ts'; import { Storages } from '@/storages.ts'; import { refreshAuthorStats } from '@/utils/stats.ts'; @@ -18,6 +17,6 @@ try { } const store = await Storages.db(); -const { kysely } = await DittoDB.getInstance(); +const kysely = await Storages.kysely(); await refreshAuthorStats({ pubkey, kysely, store }); diff --git a/src/controllers/api/oauth.ts b/src/controllers/api/oauth.ts index a736f5ca..94aaeecd 100644 --- a/src/controllers/api/oauth.ts +++ b/src/controllers/api/oauth.ts @@ -6,7 +6,6 @@ import { z } from 'zod'; import { AppController } from '@/app.ts'; import { Conf } from '@/config.ts'; -import { DittoDB } from '@/db/DittoDB.ts'; import { nostrNow } from '@/utils.ts'; import { parseBody } from '@/utils/api.ts'; import { Storages } from '@/storages.ts'; @@ -82,7 +81,7 @@ const createTokenController: AppController = async (c) => { async function getToken( { pubkey, secret, relays = [] }: { pubkey: string; secret?: string; relays?: string[] }, ): Promise<`token1${string}`> { - const { kysely } = await DittoDB.getInstance(); + const kysely = await Storages.kysely(); const token = generateToken(); const serverSeckey = generateSecretKey(); diff --git a/src/controllers/api/statuses.ts b/src/controllers/api/statuses.ts index bef98122..e0956e74 100644 --- a/src/controllers/api/statuses.ts +++ b/src/controllers/api/statuses.ts @@ -8,7 +8,6 @@ import { z } from 'zod'; import { type AppController } from '@/app.ts'; import { Conf } from '@/config.ts'; -import { DittoDB } from '@/db/DittoDB.ts'; import { DittoUpload, dittoUploads } from '@/DittoUploads.ts'; import { DittoEvent } from '@/interfaces/DittoEvent.ts'; import { getAncestors, getAuthor, getDescendants, getEvent } from '@/queries.ts'; @@ -579,7 +578,7 @@ const zappedByController: AppController = async (c) => { const id = c.req.param('id'); const params = c.get('listPagination'); const store = await Storages.db(); - const { kysely } = await DittoDB.getInstance(); + const kysely = await Storages.kysely(); const zaps = await kysely.selectFrom('event_zaps') .selectAll() diff --git a/src/controllers/api/streaming.ts b/src/controllers/api/streaming.ts index 047aa573..cfa8c3c5 100644 --- a/src/controllers/api/streaming.ts +++ b/src/controllers/api/streaming.ts @@ -4,7 +4,6 @@ import { z } from 'zod'; import { type AppController } from '@/app.ts'; import { Conf } from '@/config.ts'; -import { DittoDB } from '@/db/DittoDB.ts'; import { streamingConnectionsGauge } from '@/metrics.ts'; import { MuteListPolicy } from '@/policies/MuteListPolicy.ts'; import { getFeedPubkeys } from '@/queries.ts'; @@ -222,7 +221,7 @@ async function topicToFilter( async function getTokenPubkey(token: string): Promise { if (token.startsWith('token1')) { - const { kysely } = await DittoDB.getInstance(); + const kysely = await Storages.kysely(); const { user_pubkey } = await kysely .selectFrom('nip46_tokens') diff --git a/src/controllers/metrics.ts b/src/controllers/metrics.ts index e25522ff..4ef378a0 100644 --- a/src/controllers/metrics.ts +++ b/src/controllers/metrics.ts @@ -1,12 +1,12 @@ import { register } from 'prom-client'; import { AppController } from '@/app.ts'; -import { DittoDB } from '@/db/DittoDB.ts'; import { dbAvailableConnectionsGauge, dbPoolSizeGauge } from '@/metrics.ts'; +import { Storages } from '@/storages.ts'; /** Prometheus/OpenMetrics controller. */ export const metricsController: AppController = async (c) => { - const db = await DittoDB.getInstance(); + const db = await Storages.database(); // Update some metrics at request time. dbPoolSizeGauge.set(db.poolSize); diff --git a/src/db/DittoDB.ts b/src/db/DittoDB.ts index 63d2bfb1..445c3da2 100644 --- a/src/db/DittoDB.ts +++ b/src/db/DittoDB.ts @@ -3,24 +3,12 @@ import path from 'node:path'; import { FileMigrationProvider, Kysely, Migrator } from 'kysely'; -import { Conf } from '@/config.ts'; import { DittoPglite } from '@/db/adapters/DittoPglite.ts'; import { DittoPostgres } from '@/db/adapters/DittoPostgres.ts'; import { DittoDatabase, DittoDatabaseOpts } from '@/db/DittoDatabase.ts'; import { DittoTables } from '@/db/DittoTables.ts'; export class DittoDB { - private static db: DittoDatabase | undefined; - - /** Create (and migrate) the database if it isn't been already, or return the existing connection. */ - static async getInstance(): Promise { - if (!this.db) { - this.db = this.create(Conf.databaseUrl, { poolSize: Conf.pg.poolSize }); - await this.migrate(this.db.kysely); - } - return this.db; - } - /** Open a new database connection. */ static create(databaseUrl: string, opts?: DittoDatabaseOpts): DittoDatabase { const { protocol } = new URL(databaseUrl); diff --git a/src/middleware/signerMiddleware.ts b/src/middleware/signerMiddleware.ts index 60826db9..344e14ef 100644 --- a/src/middleware/signerMiddleware.ts +++ b/src/middleware/signerMiddleware.ts @@ -5,7 +5,7 @@ import { nip19 } from 'nostr-tools'; import { AppMiddleware } from '@/app.ts'; import { ConnectSigner } from '@/signers/ConnectSigner.ts'; import { ReadOnlySigner } from '@/signers/ReadOnlySigner.ts'; -import { DittoDB } from '@/db/DittoDB.ts'; +import { Storages } from '@/storages.ts'; /** We only accept "Bearer" type. */ const BEARER_REGEX = new RegExp(`^Bearer (${nip19.BECH32_REGEX.source})$`); @@ -20,7 +20,7 @@ export const signerMiddleware: AppMiddleware = async (c, next) => { if (bech32.startsWith('token1')) { try { - const { kysely } = await DittoDB.getInstance(); + const kysely = await Storages.kysely(); const { user_pubkey, server_seckey, relays } = await kysely .selectFrom('nip46_tokens') diff --git a/src/pipeline.ts b/src/pipeline.ts index dd59cb8d..88e5f29b 100644 --- a/src/pipeline.ts +++ b/src/pipeline.ts @@ -5,7 +5,6 @@ import { LRUCache } from 'lru-cache'; import { z } from 'zod'; import { Conf } from '@/config.ts'; -import { DittoDB } from '@/db/DittoDB.ts'; import { DittoEvent } from '@/interfaces/DittoEvent.ts'; import { pipelineEventsCounter, policyEventsCounter } from '@/metrics.ts'; import { RelayError } from '@/RelayError.ts'; @@ -53,7 +52,7 @@ async function handleEvent(event: DittoEvent, signal: AbortSignal): Promise { async function hydrateEvent(event: DittoEvent, signal: AbortSignal): Promise { await hydrateEvents({ events: [event], store: await Storages.db(), signal }); - const { kysely } = await DittoDB.getInstance(); + const kysely = await Storages.kysely(); const domain = await kysely .selectFrom('pubkey_domains') .select('domain') @@ -118,7 +117,7 @@ async function hydrateEvent(event: DittoEvent, signal: AbortSignal): Promise { if (NKinds.ephemeral(event.kind)) return; const store = await Storages.db(); - const { kysely } = await DittoDB.getInstance(); + const kysely = await Storages.kysely(); await updateStats({ event, store, kysely }).catch(debug); await store.event(event, { signal }); @@ -146,7 +145,7 @@ async function parseMetadata(event: NostrEvent, signal: AbortSignal): Promise | undefined; + private static _database: DittoDatabase | undefined; private static _admin: Promise | undefined; private static _client: Promise | undefined; private static _pubsub: Promise | undefined; private static _search: Promise | undefined; + public static async database(): Promise { + if (!this._database) { + this._database = DittoDB.create(Conf.databaseUrl, { poolSize: Conf.pg.poolSize }); + await DittoDB.migrate(this._database.kysely); + } + return this._database; + } + + public static async kysely(): Promise { + const { kysely } = await this.database(); + return kysely; + } + /** SQL database to store events this Ditto server cares about. */ public static async db(): Promise { if (!this._db) { this._db = (async () => { - const { kysely } = await DittoDB.getInstance(); + const { kysely } = await this.database(); const store = new EventsDB({ kysely, pubkey: Conf.pubkey, timeout: Conf.db.timeouts.default }); await seedZapSplits(store); return store; diff --git a/src/storages/hydrate.ts b/src/storages/hydrate.ts index c7a277c7..7b11cfb8 100644 --- a/src/storages/hydrate.ts +++ b/src/storages/hydrate.ts @@ -1,13 +1,13 @@ import { NStore } from '@nostrify/nostrify'; +import { Kysely } from 'kysely'; import { matchFilter } from 'nostr-tools'; -import { DittoDB } from '@/db/DittoDB.ts'; import { DittoTables } from '@/db/DittoTables.ts'; import { Conf } from '@/config.ts'; import { type DittoEvent } from '@/interfaces/DittoEvent.ts'; import { findQuoteTag } from '@/utils/tags.ts'; import { findQuoteInContent } from '@/utils/note.ts'; -import { Kysely } from 'kysely'; +import { Storages } from '@/storages.ts'; interface HydrateOpts { events: DittoEvent[]; @@ -18,7 +18,7 @@ interface HydrateOpts { /** Hydrate events using the provided storage. */ async function hydrateEvents(opts: HydrateOpts): Promise { - const { events, store, signal, kysely = (await DittoDB.getInstance()).kysely } = opts; + const { events, store, signal, kysely = await Storages.kysely() } = opts; if (!events.length) { return events; diff --git a/src/trends.ts b/src/trends.ts index 337ee5c7..de91a33d 100644 --- a/src/trends.ts +++ b/src/trends.ts @@ -3,10 +3,10 @@ import { Stickynotes } from '@soapbox/stickynotes'; import { Kysely, sql } from 'kysely'; import { Conf } from '@/config.ts'; -import { DittoDB } from '@/db/DittoDB.ts'; import { DittoTables } from '@/db/DittoTables.ts'; import { handleEvent } from '@/pipeline.ts'; import { AdminSigner } from '@/signers/AdminSigner.ts'; +import { Storages } from '@/storages.ts'; import { Time } from '@/utils/time.ts'; const console = new Stickynotes('ditto:trends'); @@ -70,7 +70,7 @@ export async function updateTrendingTags( aliases?: string[], ) { console.info(`Updating trending ${l}...`); - const { kysely } = await DittoDB.getInstance(); + const kysely = await Storages.kysely(); const signal = AbortSignal.timeout(1000); const yesterday = Math.floor((Date.now() - Time.days(1)) / 1000); From cae0f492f3dac0bb3b428312c99ffae3efb1307a Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Wed, 11 Sep 2024 14:04:11 -0500 Subject: [PATCH 38/82] Let PolicyWorker run in sandbox with store --- src/config.ts | 9 +++++++++ src/workers/policy.ts | 11 ++++++++--- src/workers/policy.worker.ts | 18 +++++++++++++++++- 3 files changed, 34 insertions(+), 4 deletions(-) diff --git a/src/config.ts b/src/config.ts index 5fa7be9a..37b1d0f3 100644 --- a/src/config.ts +++ b/src/config.ts @@ -1,3 +1,4 @@ +import os from 'node:os'; import * as dotenv from '@std/dotenv'; import { getPublicKey, nip19 } from 'nostr-tools'; import { z } from 'zod'; @@ -240,6 +241,14 @@ class Conf { static get policy(): string { return Deno.env.get('DITTO_POLICY') || new URL('../data/policy.ts', import.meta.url).pathname; } + /** Absolute path to the data directory used by Ditto. */ + static get dataDir(): string { + return Deno.env.get('DITTO_DATA_DIR') || new URL('../data', import.meta.url).pathname; + } + /** Absolute path of the Deno directory. */ + static get denoDir(): string { + return Deno.env.get('DENO_DIR') || `${os.userInfo().homedir}/.cache/deno`; + } /** Whether zap splits should be enabled. */ static get zapSplitsEnabled(): boolean { return optionalBooleanSchema.parse(Deno.env.get('ZAP_SPLITS_ENABLED')) ?? false; diff --git a/src/workers/policy.ts b/src/workers/policy.ts index 08511ded..f86f9d9b 100644 --- a/src/workers/policy.ts +++ b/src/workers/policy.ts @@ -13,8 +13,8 @@ export const policyWorker = Comlink.wrap( type: 'module', deno: { permissions: { - read: [Conf.policy], - write: false, + read: [Conf.denoDir, Conf.policy, Conf.dataDir], + write: [Conf.dataDir], net: 'inherit', env: false, }, @@ -24,7 +24,12 @@ export const policyWorker = Comlink.wrap( ); try { - await policyWorker.init(Conf.policy, Conf.databaseUrl, Conf.pubkey); + await policyWorker.init({ + path: Conf.policy, + cwd: Deno.cwd(), + databaseUrl: Conf.databaseUrl, + adminPubkey: Conf.pubkey, + }); console.debug(`Using custom policy: ${Conf.policy}`); } catch (e) { if (e.message.includes('Module not found')) { diff --git a/src/workers/policy.worker.ts b/src/workers/policy.worker.ts index 0036c4bd..3fb4ef3f 100644 --- a/src/workers/policy.worker.ts +++ b/src/workers/policy.worker.ts @@ -6,6 +6,18 @@ import * as Comlink from 'comlink'; import { DittoDB } from '@/db/DittoDB.ts'; import { EventsDB } from '@/storages/EventsDB.ts'; +/** Serializable object the worker can use to set up the state. */ +interface PolicyInit { + /** Path to the policy module (https, jsr, file, etc) */ + path: string; + /** Current working directory. */ + cwd: string; + /** Database URL to connect to. */ + databaseUrl: string; + /** Admin pubkey to use for EventsDB checks. */ + adminPubkey: string; +} + export class CustomPolicy implements NPolicy { private policy: NPolicy = new ReadOnlyPolicy(); @@ -14,7 +26,11 @@ export class CustomPolicy implements NPolicy { return this.policy.call(event); } - async init(path: string, databaseUrl: string, adminPubkey: string): Promise { + async init({ path, cwd, databaseUrl, adminPubkey }: PolicyInit): Promise { + // HACK: PGlite uses `path.resolve`, which requires read permission on Deno (which we don't want to give). + // We can work around this getting the cwd from the caller and overwriting `Deno.cwd`. + Deno.cwd = () => cwd; + const { kysely } = DittoDB.create(databaseUrl, { poolSize: 1 }); const store = new EventsDB({ From 6b66e61ee94bb6ee4704eb4bd5692d3f1f310801 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Wed, 11 Sep 2024 14:34:41 -0500 Subject: [PATCH 39/82] PolicyWorker: mock Deno.env --- src/workers/policy.worker.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/workers/policy.worker.ts b/src/workers/policy.worker.ts index 3fb4ef3f..1d65f405 100644 --- a/src/workers/policy.worker.ts +++ b/src/workers/policy.worker.ts @@ -6,6 +6,9 @@ import * as Comlink from 'comlink'; import { DittoDB } from '@/db/DittoDB.ts'; import { EventsDB } from '@/storages/EventsDB.ts'; +// @ts-ignore Don't try to access the env from this worker. +Deno.env = new Map(); + /** Serializable object the worker can use to set up the state. */ interface PolicyInit { /** Path to the policy module (https, jsr, file, etc) */ From b449b17fbe8d6996c3954c3662bcdd66e71da912 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Wed, 11 Sep 2024 16:11:09 -0500 Subject: [PATCH 40/82] Add updated gleasonator-policy to deno.lock --- deno.lock | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/deno.lock b/deno.lock index ca1f8be3..83c5dde0 100644 --- a/deno.lock +++ b/deno.lock @@ -11,6 +11,7 @@ "jsr:@gleasonator/policy@0.4.1": "jsr:@gleasonator/policy@0.4.1", "jsr:@gleasonator/policy@0.4.2": "jsr:@gleasonator/policy@0.4.2", "jsr:@gleasonator/policy@0.5.0": "jsr:@gleasonator/policy@0.5.0", + "jsr:@gleasonator/policy@0.5.1": "jsr:@gleasonator/policy@0.5.1", "jsr:@hono/hono@^4.4.6": "jsr:@hono/hono@4.5.11", "jsr:@lambdalisue/async@^2.1.1": "jsr:@lambdalisue/async@2.1.1", "jsr:@nostrify/db@^0.31.2": "jsr:@nostrify/db@0.31.2", @@ -146,6 +147,13 @@ "jsr:@nostrify/policies@^0.33.0" ] }, + "@gleasonator/policy@0.5.1": { + "integrity": "2d687c5166556ce13ac05c4542f61ef8a47d8b96b57f6e43d52035805f895551", + "dependencies": [ + "jsr:@nostrify/nostrify@^0.31.0", + "jsr:@nostrify/policies@^0.33.0" + ] + }, "@hono/hono@4.4.6": { "integrity": "aa557ca9930787ee86b9ca1730691f1ce1c379174c2cb244d5934db2b6314453" }, From 20d1f885ac6576c6569b7a257238324026cb01c2 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Wed, 11 Sep 2024 16:53:02 -0500 Subject: [PATCH 41/82] Upgrade gleasonator-policy in deno.lock --- deno.lock | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/deno.lock b/deno.lock index 83c5dde0..a275accb 100644 --- a/deno.lock +++ b/deno.lock @@ -12,6 +12,7 @@ "jsr:@gleasonator/policy@0.4.2": "jsr:@gleasonator/policy@0.4.2", "jsr:@gleasonator/policy@0.5.0": "jsr:@gleasonator/policy@0.5.0", "jsr:@gleasonator/policy@0.5.1": "jsr:@gleasonator/policy@0.5.1", + "jsr:@gleasonator/policy@0.5.2": "jsr:@gleasonator/policy@0.5.2", "jsr:@hono/hono@^4.4.6": "jsr:@hono/hono@4.5.11", "jsr:@lambdalisue/async@^2.1.1": "jsr:@lambdalisue/async@2.1.1", "jsr:@nostrify/db@^0.31.2": "jsr:@nostrify/db@0.31.2", @@ -22,6 +23,7 @@ "jsr:@nostrify/nostrify@^0.30.1": "jsr:@nostrify/nostrify@0.30.1", "jsr:@nostrify/nostrify@^0.31.0": "jsr:@nostrify/nostrify@0.31.0", "jsr:@nostrify/policies@^0.33.0": "jsr:@nostrify/policies@0.33.0", + "jsr:@nostrify/policies@^0.33.1": "jsr:@nostrify/policies@0.33.1", "jsr:@nostrify/types@^0.30.0": "jsr:@nostrify/types@0.30.1", "jsr:@nostrify/types@^0.30.1": "jsr:@nostrify/types@0.30.1", "jsr:@soapbox/kysely-pglite@^0.0.1": "jsr:@soapbox/kysely-pglite@0.0.1", @@ -154,6 +156,13 @@ "jsr:@nostrify/policies@^0.33.0" ] }, + "@gleasonator/policy@0.5.2": { + "integrity": "cdd3add87be3132eb05736bca640dfb3bbb1aa79928a44d3563cde20bab7c0d3", + "dependencies": [ + "jsr:@nostrify/nostrify@^0.31.0", + "jsr:@nostrify/policies@^0.33.1" + ] + }, "@hono/hono@4.4.6": { "integrity": "aa557ca9930787ee86b9ca1730691f1ce1c379174c2cb244d5934db2b6314453" }, @@ -269,6 +278,13 @@ "npm:nostr-tools@^2.7.0" ] }, + "@nostrify/policies@0.33.1": { + "integrity": "381e1f9406a6da22da03a254e46b1aa07d5491b9761961cda3a4aeb5bf3f5286", + "dependencies": [ + "jsr:@nostrify/types@^0.30.1", + "npm:nostr-tools@^2.7.0" + ] + }, "@nostrify/types@0.30.0": { "integrity": "1f38fa849cff930bd709edbf94ef9ac02f46afb8b851f86c8736517b354616da" }, From f7ee59146e28b2b986e48fd610a4dddcbe2d4eca Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Wed, 11 Sep 2024 18:30:23 -0500 Subject: [PATCH 42/82] Clean up config --- src/config.ts | 20 ++------------------ src/controllers/api/instance.ts | 3 --- 2 files changed, 2 insertions(+), 21 deletions(-) diff --git a/src/config.ts b/src/config.ts index 37b1d0f3..e174c81a 100644 --- a/src/config.ts +++ b/src/config.ts @@ -35,21 +35,11 @@ class Conf { } return this._pubkey; } - /** Ditto admin secret key as a Web Crypto key. */ - static get cryptoKey(): Promise { - return crypto.subtle.importKey( - 'raw', - Conf.seckey, - { name: 'HMAC', hash: 'SHA-256' }, - false, - ['sign', 'verify'], - ); - } - + /** Port to use when serving the HTTP server. */ static get port(): number { return parseInt(Deno.env.get('PORT') || '4036'); } - + /** Relay URL to the Ditto server's relay. */ static get relay(): `wss://${string}` | `ws://${string}` { const { protocol, host } = Conf.url; return `${protocol === 'https:' ? 'wss:' : 'ws:'}//${host}/relay`; @@ -188,12 +178,6 @@ class Conf { 'system', ]; } - /** Proof-of-work configuration. */ - static pow = { - get registrations(): number { - return Number(Deno.env.get('DITTO_POW_REGISTRATIONS') ?? 20); - }, - }; /** Domain of the Ditto server as a `URL` object, for easily grabbing the `hostname`, etc. */ static get url(): URL { return new URL(Conf.localDomain); diff --git a/src/controllers/api/instance.ts b/src/controllers/api/instance.ts index faca9c9f..b350963d 100644 --- a/src/controllers/api/instance.ts +++ b/src/controllers/api/instance.ts @@ -63,9 +63,6 @@ const instanceV1Controller: AppController = async (c) => { nostr: { pubkey: Conf.pubkey, relay: `${wsProtocol}//${host}/relay`, - pow: { - registrations: Conf.pow.registrations, - }, }, rules: [], }); From 593cedba0623fd62b047d7e17b52487583061e91 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Wed, 11 Sep 2024 21:34:16 -0500 Subject: [PATCH 43/82] grafana: add API endpoints row --- grafana/Ditto-Dashboard.json | 2474 +++++++++++++++++++++++++++++++++- 1 file changed, 2454 insertions(+), 20 deletions(-) diff --git a/grafana/Ditto-Dashboard.json b/grafana/Ditto-Dashboard.json index 0c161eb4..4b632806 100644 --- a/grafana/Ditto-Dashboard.json +++ b/grafana/Ditto-Dashboard.json @@ -35,7 +35,7 @@ "type": "grafana", "id": "grafana", "name": "Grafana", - "version": "11.1.4" + "version": "11.2.0" }, { "type": "datasource", @@ -123,6 +123,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 25, "gradientMode": "none", @@ -363,7 +364,7 @@ "showThresholdMarkers": true, "sizing": "auto" }, - "pluginVersion": "11.1.4", + "pluginVersion": "11.2.0", "targets": [ { "datasource": { @@ -437,7 +438,7 @@ "sizing": "auto", "valueMode": "color" }, - "pluginVersion": "11.1.4", + "pluginVersion": "11.2.0", "targets": [ { "datasource": { @@ -486,6 +487,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -583,6 +585,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -680,6 +683,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", @@ -822,6 +826,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -903,13 +908,2440 @@ "type": "timeseries" }, { - "collapsed": false, + "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 27 }, + "id": 27, + "panels": [ + { + "datasource": { + "default": false, + "type": "prometheus", + "uid": "${prometheus}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 25, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "Server Errors" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "semi-dark-red", + "mode": "shades" + } + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "Client Errors" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "dark-blue", + "mode": "shades" + } + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "Redirects" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "semi-dark-yellow", + "mode": "shades" + } + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "Success" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "semi-dark-green", + "mode": "shades" + } + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 0, + "y": 28 + }, + "id": 34, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": false, + "width": 225 + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "increase(ditto_http_responses_total{status=~\"5..\",method=\"GET\",path=\"/api/v1/timelines/home\"}[$__rate_interval])", + "hide": false, + "instant": false, + "legendFormat": "{{method}} {{path}} ({{status}})", + "range": true, + "refId": "Server Errors" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheus}" + }, + "editorMode": "code", + "expr": "increase(ditto_http_responses_total{status=~\"4..\",method=\"GET\",path=\"/api/v1/timelines/home\"}[$__rate_interval])", + "hide": false, + "instant": false, + "legendFormat": "{{method}} {{path}} ({{status}})", + "range": true, + "refId": "Client Errors" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "increase(ditto_http_responses_total{status=~\"3..\",method=\"GET\",path=\"/api/v1/timelines/home\"}[$__rate_interval])", + "hide": false, + "instant": false, + "legendFormat": "{{method}} {{path}} ({{status}})", + "range": true, + "refId": "Redirects" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheus}" + }, + "editorMode": "code", + "expr": "increase(ditto_http_responses_total{status=~\"2..\",method=\"GET\",path=\"/api/v1/timelines/home\"}[$__rate_interval])", + "instant": false, + "interval": "", + "legendFormat": "{{method}} {{path}} ({{status}})", + "range": true, + "refId": "Success" + } + ], + "title": "GET /api/v1/timelines/home", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 25, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "Server Errors" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "semi-dark-red", + "mode": "shades" + } + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "Client Errors" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "dark-blue", + "mode": "shades" + } + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "Redirects" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "semi-dark-yellow", + "mode": "shades" + } + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "Success" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "semi-dark-green", + "mode": "shades" + } + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 6, + "y": 28 + }, + "id": 25, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": false, + "width": 225 + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "increase(ditto_http_responses_total{status=~\"5..\",method=\"GET\",path=\"/api/v1/notifications\"}[$__rate_interval])", + "hide": false, + "instant": false, + "legendFormat": "{{method}} {{path}} ({{status}})", + "range": true, + "refId": "Server Errors" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheus}" + }, + "editorMode": "code", + "expr": "increase(ditto_http_responses_total{status=~\"4..\",method=\"GET\",path=\"/api/v1/notifications\"}[$__rate_interval])", + "hide": false, + "instant": false, + "legendFormat": "{{method}} {{path}} ({{status}})", + "range": true, + "refId": "Client Errors" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "increase(ditto_http_responses_total{status=~\"3..\",method=\"GET\",path=\"/api/v1/notifications\"}[$__rate_interval])", + "hide": false, + "instant": false, + "legendFormat": "{{method}} {{path}} ({{status}})", + "range": true, + "refId": "Redirects" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheus}" + }, + "editorMode": "code", + "expr": "increase(ditto_http_responses_total{status=~\"2..\",method=\"GET\",path=\"/api/v1/notifications\"}[$__rate_interval])", + "instant": false, + "interval": "", + "legendFormat": "{{method}} {{path}} ({{status}})", + "range": true, + "refId": "Success" + } + ], + "title": "GET /api/v1/notifications", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 25, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "Server Errors" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "semi-dark-red", + "mode": "shades" + } + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "Client Errors" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "dark-blue", + "mode": "shades" + } + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "Redirects" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "semi-dark-yellow", + "mode": "shades" + } + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "Success" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "semi-dark-green", + "mode": "shades" + } + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 12, + "y": 28 + }, + "id": 31, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": false, + "width": 225 + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "increase(ditto_http_responses_total{status=~\"5..\",method=\"GET\",path=\"/api/v1/accounts/verify_credentials\"}[$__rate_interval])", + "hide": false, + "instant": false, + "legendFormat": "{{method}} {{path}} ({{status}})", + "range": true, + "refId": "Server Errors" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheus}" + }, + "editorMode": "code", + "expr": "increase(ditto_http_responses_total{status=~\"4..\",method=\"GET\",path=\"/api/v1/accounts/verify_credentials\"}[$__rate_interval])", + "hide": false, + "instant": false, + "legendFormat": "{{method}} {{path}} ({{status}})", + "range": true, + "refId": "Client Errors" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "increase(ditto_http_responses_total{status=~\"3..\",method=\"GET\",path=\"/api/v1/accounts/verify_credentials\"}[$__rate_interval])", + "hide": false, + "instant": false, + "legendFormat": "{{method}} {{path}} ({{status}})", + "range": true, + "refId": "Redirects" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheus}" + }, + "editorMode": "code", + "expr": "increase(ditto_http_responses_total{status=~\"2..\",method=\"GET\",path=\"/api/v1/accounts/verify_credentials\"}[$__rate_interval])", + "instant": false, + "interval": "", + "legendFormat": "{{method}} {{path}} ({{status}})", + "range": true, + "refId": "Success" + } + ], + "title": "GET /api/v1/accounts/verify_credentials", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 25, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "Server Errors" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "semi-dark-red", + "mode": "shades" + } + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "Client Errors" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "dark-blue", + "mode": "shades" + } + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "Redirects" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "semi-dark-yellow", + "mode": "shades" + } + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "Success" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "semi-dark-green", + "mode": "shades" + } + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 18, + "y": 28 + }, + "id": 32, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": false, + "width": 225 + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "increase(ditto_http_responses_total{status=~\"5..\",method=\"POST\",path=\"/oauth/token\"}[$__rate_interval])", + "hide": false, + "instant": false, + "legendFormat": "{{method}} {{path}} ({{status}})", + "range": true, + "refId": "Server Errors" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheus}" + }, + "editorMode": "code", + "expr": "increase(ditto_http_responses_total{status=~\"4..\",method=\"POST\",path=\"/oauth/token\"}[$__rate_interval])", + "hide": false, + "instant": false, + "legendFormat": "{{method}} {{path}} ({{status}})", + "range": true, + "refId": "Client Errors" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "increase(ditto_http_responses_total{status=~\"3..\",method=\"POST\",path=\"/oauth/token\"}[$__rate_interval])", + "hide": false, + "instant": false, + "legendFormat": "{{method}} {{path}} ({{status}})", + "range": true, + "refId": "Redirects" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheus}" + }, + "editorMode": "code", + "expr": "increase(ditto_http_responses_total{status=~\"2..\",method=\"POST\",path=\"/oauth/token\"}[$__rate_interval])", + "instant": false, + "interval": "", + "legendFormat": "{{method}} {{path}} ({{status}})", + "range": true, + "refId": "Success" + } + ], + "title": "POST /oauth/token", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 25, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "Server Errors" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "semi-dark-red", + "mode": "shades" + } + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "Client Errors" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "dark-blue", + "mode": "shades" + } + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "Redirects" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "semi-dark-yellow", + "mode": "shades" + } + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "Success" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "semi-dark-green", + "mode": "shades" + } + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 0, + "y": 35 + }, + "id": 30, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": false, + "width": 225 + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "increase(ditto_http_responses_total{status=~\"5..\",method=\"GET\",path=\"/api/v1/instance\"}[$__rate_interval])", + "hide": false, + "instant": false, + "legendFormat": "{{method}} {{path}} ({{status}})", + "range": true, + "refId": "Server Errors" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheus}" + }, + "editorMode": "code", + "expr": "increase(ditto_http_responses_total{status=~\"4..\",method=\"GET\",path=\"/api/v1/instance\"}[$__rate_interval])", + "hide": false, + "instant": false, + "legendFormat": "{{method}} {{path}} ({{status}})", + "range": true, + "refId": "Client Errors" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "increase(ditto_http_responses_total{status=~\"3..\",method=\"GET\",path=\"/api/v1/instance\"}[$__rate_interval])", + "hide": false, + "instant": false, + "legendFormat": "{{method}} {{path}} ({{status}})", + "range": true, + "refId": "Redirects" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheus}" + }, + "editorMode": "code", + "expr": "increase(ditto_http_responses_total{status=~\"2..\",method=\"GET\",path=\"/api/v1/instance\"}[$__rate_interval])", + "instant": false, + "interval": "", + "legendFormat": "{{method}} {{path}} ({{status}})", + "range": true, + "refId": "Success" + } + ], + "title": "GET /api/v1/instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 25, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "Server Errors" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "semi-dark-red", + "mode": "shades" + } + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "Client Errors" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "dark-blue", + "mode": "shades" + } + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "Redirects" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "semi-dark-yellow", + "mode": "shades" + } + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "Success" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "semi-dark-green", + "mode": "shades" + } + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 6, + "y": 35 + }, + "id": 35, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": false, + "width": 225 + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "increase(ditto_http_responses_total{status=~\"5..\",method=\"GET\",path=\"/api/v1/timelines/public\"}[$__rate_interval])", + "hide": false, + "instant": false, + "legendFormat": "{{method}} {{path}} ({{status}})", + "range": true, + "refId": "Server Errors" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheus}" + }, + "editorMode": "code", + "expr": "increase(ditto_http_responses_total{status=~\"4..\",method=\"GET\",path=\"/api/v1/timelines/public\"}[$__rate_interval])", + "hide": false, + "instant": false, + "legendFormat": "{{method}} {{path}} ({{status}})", + "range": true, + "refId": "Client Errors" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "increase(ditto_http_responses_total{status=~\"3..\",method=\"GET\",path=\"/api/v1/timelines/public\"}[$__rate_interval])", + "hide": false, + "instant": false, + "legendFormat": "{{method}} {{path}} ({{status}})", + "range": true, + "refId": "Redirects" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheus}" + }, + "editorMode": "code", + "expr": "increase(ditto_http_responses_total{status=~\"2..\",method=\"GET\",path=\"/api/v1/timelines/public\"}[$__rate_interval])", + "instant": false, + "interval": "", + "legendFormat": "{{method}} {{path}} ({{status}})", + "range": true, + "refId": "Success" + } + ], + "title": "GET /api/v1/timelines/public", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 25, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "Server Errors" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "semi-dark-red", + "mode": "shades" + } + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "Client Errors" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "dark-blue", + "mode": "shades" + } + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "Redirects" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "semi-dark-yellow", + "mode": "shades" + } + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "Success" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "semi-dark-green", + "mode": "shades" + } + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 12, + "y": 35 + }, + "id": 36, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": false, + "width": 225 + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "increase(ditto_http_responses_total{status=~\"5..\",method=\"GET\",path=\"/api/v1/timelines/tag/:hashtag\"}[$__rate_interval])", + "hide": false, + "instant": false, + "legendFormat": "{{method}} {{path}} ({{status}})", + "range": true, + "refId": "Server Errors" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheus}" + }, + "editorMode": "code", + "expr": "increase(ditto_http_responses_total{status=~\"4..\",method=\"GET\",path=\"/api/v1/timelines/tag/:hashtag\"}[$__rate_interval])", + "hide": false, + "instant": false, + "legendFormat": "{{method}} {{path}} ({{status}})", + "range": true, + "refId": "Client Errors" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "increase(ditto_http_responses_total{status=~\"3..\",method=\"GET\",path=\"/api/v1/timelines/tag/:hashtag\"}[$__rate_interval])", + "hide": false, + "instant": false, + "legendFormat": "{{method}} {{path}} ({{status}})", + "range": true, + "refId": "Redirects" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheus}" + }, + "editorMode": "code", + "expr": "increase(ditto_http_responses_total{status=~\"2..\",method=\"GET\",path=\"/api/v1/timelines/tag/:hashtag\"}[$__rate_interval])", + "instant": false, + "interval": "", + "legendFormat": "{{method}} {{path}} ({{status}})", + "range": true, + "refId": "Success" + } + ], + "title": "GET /api/v1/timelines/tag/:hashtag", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 25, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "Server Errors" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "semi-dark-red", + "mode": "shades" + } + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "Client Errors" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "dark-blue", + "mode": "shades" + } + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "Redirects" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "semi-dark-yellow", + "mode": "shades" + } + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "Success" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "semi-dark-green", + "mode": "shades" + } + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 18, + "y": 35 + }, + "id": 28, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": false, + "width": 225 + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "increase(ditto_http_responses_total{status=~\"5..\",method=\"GET\",path=\"/api/v2/search\"}[$__rate_interval])", + "hide": false, + "instant": false, + "legendFormat": "{{method}} {{path}} ({{status}})", + "range": true, + "refId": "Server Errors" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheus}" + }, + "editorMode": "code", + "expr": "increase(ditto_http_responses_total{status=~\"4..\",method=\"GET\",path=\"/api/v2/search\"}[$__rate_interval])", + "hide": false, + "instant": false, + "legendFormat": "{{method}} {{path}} ({{status}})", + "range": true, + "refId": "Client Errors" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "increase(ditto_http_responses_total{status=~\"3..\",method=\"GET\",path=\"/api/v2/search\"}[$__rate_interval])", + "hide": false, + "instant": false, + "legendFormat": "{{method}} {{path}} ({{status}})", + "range": true, + "refId": "Redirects" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheus}" + }, + "editorMode": "code", + "expr": "increase(ditto_http_responses_total{status=~\"2..\",method=\"GET\",path=\"/api/v2/search\"}[$__rate_interval])", + "instant": false, + "interval": "", + "legendFormat": "{{method}} {{path}} ({{status}})", + "range": true, + "refId": "Success" + } + ], + "title": "GET /api/v2/search", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 25, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "Server Errors" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "semi-dark-red", + "mode": "shades" + } + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "Client Errors" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "dark-blue", + "mode": "shades" + } + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "Redirects" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "semi-dark-yellow", + "mode": "shades" + } + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "Success" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "semi-dark-green", + "mode": "shades" + } + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 0, + "y": 42 + }, + "id": 26, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": false, + "width": 225 + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "increase(ditto_http_responses_total{status=~\"5..\",method=\"POST\",path=\"/api/v1/statuses\"}[$__rate_interval])", + "hide": false, + "instant": false, + "legendFormat": "{{method}} {{path}} ({{status}})", + "range": true, + "refId": "Server Errors" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheus}" + }, + "editorMode": "code", + "expr": "increase(ditto_http_responses_total{status=~\"4..\",method=\"POST\",path=\"/api/v1/statuses\"}[$__rate_interval])", + "hide": false, + "instant": false, + "legendFormat": "{{method}} {{path}} ({{status}})", + "range": true, + "refId": "Client Errors" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "increase(ditto_http_responses_total{status=~\"3..\",method=\"POST\",path=\"/api/v1/statuses\"}[$__rate_interval])", + "hide": false, + "instant": false, + "legendFormat": "{{method}} {{path}} ({{status}})", + "range": true, + "refId": "Redirects" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheus}" + }, + "editorMode": "code", + "expr": "increase(ditto_http_responses_total{status=~\"2..\",method=\"POST\",path=\"/api/v1/statuses\"}[$__rate_interval])", + "instant": false, + "interval": "", + "legendFormat": "{{method}} {{path}} ({{status}})", + "range": true, + "refId": "Success" + } + ], + "title": "POST /api/v1/statuses", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 25, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "Server Errors" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "semi-dark-red", + "mode": "shades" + } + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "Client Errors" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "dark-blue", + "mode": "shades" + } + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "Redirects" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "semi-dark-yellow", + "mode": "shades" + } + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "Success" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "semi-dark-green", + "mode": "shades" + } + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 6, + "y": 42 + }, + "id": 29, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": false, + "width": 225 + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "increase(ditto_http_responses_total{status=~\"5..\",method=\"POST\",path=\"/api/v1/statuses/:id{[0-9a-f]{64}}/favourite\"}[$__rate_interval])", + "hide": false, + "instant": false, + "legendFormat": "{{method}} {{path}} ({{status}})", + "range": true, + "refId": "Server Errors" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheus}" + }, + "editorMode": "code", + "expr": "increase(ditto_http_responses_total{status=~\"4..\",method=\"POST\",path=\"/api/v1/statuses/:id{[0-9a-f]{64}}/favourite\"}[$__rate_interval])", + "hide": false, + "instant": false, + "legendFormat": "{{method}} {{path}} ({{status}})", + "range": true, + "refId": "Client Errors" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "increase(ditto_http_responses_total{status=~\"3..\",method=\"POST\",path=\"/api/v1/statuses/:id{[0-9a-f]{64}}/favourite\"}[$__rate_interval])", + "hide": false, + "instant": false, + "legendFormat": "{{method}} {{path}} ({{status}})", + "range": true, + "refId": "Redirects" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheus}" + }, + "editorMode": "code", + "expr": "increase(ditto_http_responses_total{status=~\"2..\",method=\"POST\",path=\"/api/v1/statuses/:id{[0-9a-f]{64}}/favourite\"}[$__rate_interval])", + "instant": false, + "interval": "", + "legendFormat": "{{method}} {{path}} ({{status}})", + "range": true, + "refId": "Success" + } + ], + "title": "POST /api/v1/statuses/:id/favourite", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 25, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "Server Errors" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "semi-dark-red", + "mode": "shades" + } + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "Client Errors" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "dark-blue", + "mode": "shades" + } + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "Redirects" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "semi-dark-yellow", + "mode": "shades" + } + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "Success" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "semi-dark-green", + "mode": "shades" + } + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 12, + "y": 42 + }, + "id": 37, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": false, + "width": 225 + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "increase(ditto_http_responses_total{status=~\"5..\",method=\"POST\",path=\"/api/v1/statuses/:id{[0-9a-f]{64}}/reblog\"}[$__rate_interval])", + "hide": false, + "instant": false, + "legendFormat": "{{method}} {{path}} ({{status}})", + "range": true, + "refId": "Server Errors" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheus}" + }, + "editorMode": "code", + "expr": "increase(ditto_http_responses_total{status=~\"4..\",method=\"POST\",path=\"/api/v1/statuses/:id{[0-9a-f]{64}}/reblog\"}[$__rate_interval])", + "hide": false, + "instant": false, + "legendFormat": "{{method}} {{path}} ({{status}})", + "range": true, + "refId": "Client Errors" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "increase(ditto_http_responses_total{status=~\"3..\",method=\"POST\",path=\"/api/v1/statuses/:id{[0-9a-f]{64}}/reblog\"}[$__rate_interval])", + "hide": false, + "instant": false, + "legendFormat": "{{method}} {{path}} ({{status}})", + "range": true, + "refId": "Redirects" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheus}" + }, + "editorMode": "code", + "expr": "increase(ditto_http_responses_total{status=~\"2..\",method=\"POST\",path=\"/api/v1/statuses/:id{[0-9a-f]{64}}/reblog\"}[$__rate_interval])", + "instant": false, + "interval": "", + "legendFormat": "{{method}} {{path}} ({{status}})", + "range": true, + "refId": "Success" + } + ], + "title": "POST /api/v1/statuses/:id/reblog", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 25, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "Server Errors" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "semi-dark-red", + "mode": "shades" + } + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "Client Errors" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "dark-blue", + "mode": "shades" + } + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "Redirects" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "semi-dark-yellow", + "mode": "shades" + } + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "Success" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "semi-dark-green", + "mode": "shades" + } + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 18, + "y": 42 + }, + "id": 33, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": false, + "width": 225 + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "increase(ditto_http_responses_total{status=~\"5..\",method=\"POST\",path=\"/api/v1/ditto/zap\"}[$__rate_interval])", + "hide": false, + "instant": false, + "legendFormat": "{{method}} {{path}} ({{status}})", + "range": true, + "refId": "Server Errors" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheus}" + }, + "editorMode": "code", + "expr": "increase(ditto_http_responses_total{status=~\"4..\",method=\"POST\",path=\"/api/v1/ditto/zap\"}[$__rate_interval])", + "hide": false, + "instant": false, + "legendFormat": "{{method}} {{path}} ({{status}})", + "range": true, + "refId": "Client Errors" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "increase(ditto_http_responses_total{status=~\"3..\",method=\"POST\",path=\"/api/v1/ditto/zap\"}[$__rate_interval])", + "hide": false, + "instant": false, + "legendFormat": "{{method}} {{path}} ({{status}})", + "range": true, + "refId": "Redirects" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${prometheus}" + }, + "editorMode": "code", + "expr": "increase(ditto_http_responses_total{status=~\"2..\",method=\"POST\",path=\"/api/v1/ditto/zap\"}[$__rate_interval])", + "instant": false, + "interval": "", + "legendFormat": "{{method}} {{path}} ({{status}})", + "range": true, + "refId": "Success" + } + ], + "title": "POST /api/v1/ditto/zap", + "type": "timeseries" + } + ], + "title": "API", + "type": "row" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 28 + }, "id": 21, "panels": [], "title": "Database", @@ -917,9 +3349,8 @@ }, { "datasource": { - "default": false, "type": "grafana-postgresql-datasource", - "uid": "${postgres}" + "uid": "${DS_DITTO-PG}" }, "description": "SQL queries ranked by total time.", "fieldConfig": { @@ -1092,7 +3523,7 @@ "h": 11, "w": 15, "x": 0, - "y": 28 + "y": 29 }, "id": 13, "options": { @@ -1114,7 +3545,7 @@ } ] }, - "pluginVersion": "11.1.4", + "pluginVersion": "11.2.0", "targets": [ { "datasource": { @@ -1248,7 +3679,7 @@ "h": 11, "w": 9, "x": 15, - "y": 28 + "y": 29 }, "id": 14, "options": { @@ -1263,7 +3694,7 @@ }, "showHeader": true }, - "pluginVersion": "11.1.4", + "pluginVersion": "11.2.0", "targets": [ { "datasource": { @@ -1325,7 +3756,7 @@ "h": 9, "w": 7, "x": 0, - "y": 39 + "y": 40 }, "id": 16, "options": { @@ -1422,7 +3853,7 @@ "h": 9, "w": 8, "x": 7, - "y": 39 + "y": 40 }, "id": 17, "options": { @@ -1587,7 +4018,7 @@ "h": 9, "w": 9, "x": 15, - "y": 39 + "y": 40 }, "id": 18, "options": { @@ -1602,7 +4033,7 @@ }, "showHeader": true }, - "pluginVersion": "11.1.4", + "pluginVersion": "11.2.0", "targets": [ { "datasource": { @@ -1642,7 +4073,7 @@ "h": 1, "w": 24, "x": 0, - "y": 48 + "y": 49 }, "id": 23, "panels": [], @@ -1668,6 +4099,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -1778,7 +4210,7 @@ "h": 12, "w": 24, "x": 0, - "y": 49 + "y": 50 }, "id": 9, "options": { @@ -1832,6 +4264,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -1915,7 +4348,7 @@ "h": 12, "w": 24, "x": 0, - "y": 61 + "y": 62 }, "id": 5, "options": { @@ -1967,6 +4400,7 @@ "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, + "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", @@ -2013,7 +4447,7 @@ "h": 11, "w": 24, "x": 0, - "y": 73 + "y": 74 }, "id": 19, "options": { @@ -2124,13 +4558,13 @@ ] }, "time": { - "from": "now-6h", + "from": "now-3h", "to": "now" }, "timepicker": {}, "timezone": "browser", "title": "Ditto", "uid": "ddps3ap51fv28d", - "version": 4, + "version": 7, "weekStart": "" } \ No newline at end of file From 55ee8eaf74ac0034f498ea88a54a734bbaa543f1 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Thu, 12 Sep 2024 11:09:54 -0500 Subject: [PATCH 44/82] Fix stat inflation --- src/pipeline.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/pipeline.ts b/src/pipeline.ts index 88e5f29b..4153cd41 100644 --- a/src/pipeline.ts +++ b/src/pipeline.ts @@ -119,8 +119,10 @@ async function storeEvent(event: DittoEvent, signal?: AbortSignal): Promise { + await updateStats({ event, store, kysely }); + await store.event(event, { signal }); + }); } /** Parse kind 0 metadata and track indexes in the database. */ From d67f2a27ead4bac4b24990a2a7336fb26138bd30 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Thu, 12 Sep 2024 11:30:57 -0500 Subject: [PATCH 45/82] stats: use the NPostgres transaction method to avoid transactions within transactions --- src/pipeline.ts | 3 +-- src/storages/EventsDB.ts | 4 ++++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/pipeline.ts b/src/pipeline.ts index 4153cd41..afc7acdf 100644 --- a/src/pipeline.ts +++ b/src/pipeline.ts @@ -117,9 +117,8 @@ async function hydrateEvent(event: DittoEvent, signal: AbortSignal): Promise { if (NKinds.ephemeral(event.kind)) return; const store = await Storages.db(); - const kysely = await Storages.kysely(); - await kysely.transaction().execute(async (kysely) => { + await store.transaction(async (store, kysely) => { await updateStats({ event, store, kysely }); await store.event(event, { signal }); }); diff --git a/src/storages/EventsDB.ts b/src/storages/EventsDB.ts index 72cd9bb3..148a30aa 100644 --- a/src/storages/EventsDB.ts +++ b/src/storages/EventsDB.ts @@ -316,6 +316,10 @@ class EventsDB implements NStore { return filters; } + + async transaction(callback: (store: NPostgres, kysely: Kysely) => Promise): Promise { + return this.store.transaction((store, kysely) => callback(store, kysely as unknown as Kysely)); + } } export { EventsDB }; From fc912f185e9da4a943320f3ca00cbeb6c4dcb9fa Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Thu, 12 Sep 2024 13:03:23 -0500 Subject: [PATCH 46/82] Gracefully start and exit the database --- src/DittoExit.ts | 37 ++++++++++++++++++++++++++++++++ src/db/DittoDatabase.ts | 1 + src/db/adapters/DittoPglite.ts | 5 ++++- src/db/adapters/DittoPostgres.ts | 1 + src/server.ts | 13 ++++++++++- src/storages.ts | 23 ++++++++++++++++---- 6 files changed, 74 insertions(+), 6 deletions(-) create mode 100644 src/DittoExit.ts diff --git a/src/DittoExit.ts b/src/DittoExit.ts new file mode 100644 index 00000000..36201fc7 --- /dev/null +++ b/src/DittoExit.ts @@ -0,0 +1,37 @@ +import { Stickynotes } from '@soapbox/stickynotes'; + +/** + * Add cleanup tasks to this module, + * then they will automatically be called (and the program exited) after SIGINT. + */ +export class DittoExit { + private static tasks: Array<() => Promise> = []; + private static console = new Stickynotes('ditto:exit'); + + static { + Deno.addSignalListener('SIGINT', () => this.finish('SIGINT')); + Deno.addSignalListener('SIGTERM', () => this.finish('SIGTERM')); + Deno.addSignalListener('SIGHUP', () => this.finish('SIGHUP')); + Deno.addSignalListener('SIGQUIT', () => this.finish('SIGQUIT')); + Deno.addSignalListener('SIGABRT', () => this.finish('SIGABRT')); + } + + static add(task: () => Promise): void { + this.tasks.push(task); + this.console.debug(`Added cleanup task #${this.tasks.length}`); + } + + private static async cleanup(): Promise { + this.console.debug(`Running ${this.tasks.length} cleanup tasks...`); + await Promise.allSettled( + this.tasks.map((task) => task()), + ); + } + + private static async finish(signal: Deno.Signal): Promise { + this.console.debug(signal); + await this.cleanup(); + this.console.debug('Exiting gracefully.'); + Deno.exit(0); + } +} diff --git a/src/db/DittoDatabase.ts b/src/db/DittoDatabase.ts index 530d9391..93c71a90 100644 --- a/src/db/DittoDatabase.ts +++ b/src/db/DittoDatabase.ts @@ -6,6 +6,7 @@ export interface DittoDatabase { readonly kysely: Kysely; readonly poolSize: number; readonly availableConnections: number; + readonly waitReady: Promise; } export interface DittoDatabaseOpts { diff --git a/src/db/adapters/DittoPglite.ts b/src/db/adapters/DittoPglite.ts index 4ec7d8a5..9b425c4b 100644 --- a/src/db/adapters/DittoPglite.ts +++ b/src/db/adapters/DittoPglite.ts @@ -8,9 +8,11 @@ import { KyselyLogger } from '@/db/KyselyLogger.ts'; export class DittoPglite { static create(databaseUrl: string): DittoDatabase { + const pglite = new PGlite(databaseUrl); + const kysely = new Kysely({ dialect: new PgliteDialect({ - database: new PGlite(databaseUrl), + database: pglite, }), log: KyselyLogger, }); @@ -19,6 +21,7 @@ export class DittoPglite { kysely, poolSize: 1, availableConnections: 1, + waitReady: pglite.waitReady, }; } } diff --git a/src/db/adapters/DittoPostgres.ts b/src/db/adapters/DittoPostgres.ts index f1a5bcc9..0300c3e0 100644 --- a/src/db/adapters/DittoPostgres.ts +++ b/src/db/adapters/DittoPostgres.ts @@ -48,6 +48,7 @@ export class DittoPostgres { get availableConnections() { return pg.connections.idle; }, + waitReady: Promise.resolve(), }; } } diff --git a/src/server.ts b/src/server.ts index f7a33dc0..bfa240c9 100644 --- a/src/server.ts +++ b/src/server.ts @@ -5,5 +5,16 @@ import '@/sentry.ts'; import '@/nostr-wasm.ts'; import app from '@/app.ts'; import { Conf } from '@/config.ts'; +import { DittoExit } from '@/DittoExit.ts'; -Deno.serve({ port: Conf.port }, app.fetch); +const ac = new AbortController(); +// deno-lint-ignore require-await +DittoExit.add(async () => ac.abort()); + +Deno.serve( + { + port: Conf.port, + signal: ac.signal, + }, + app.fetch, +); diff --git a/src/storages.ts b/src/storages.ts index 60a0cebc..fb66c55e 100644 --- a/src/storages.ts +++ b/src/storages.ts @@ -2,6 +2,7 @@ import { Conf } from '@/config.ts'; import { DittoDatabase } from '@/db/DittoDatabase.ts'; import { DittoDB } from '@/db/DittoDB.ts'; +import { DittoExit } from '@/DittoExit.ts'; import { AdminStore } from '@/storages/AdminStore.ts'; import { EventsDB } from '@/storages/EventsDB.ts'; import { SearchStore } from '@/storages/search-store.ts'; @@ -10,9 +11,11 @@ import { NPool, NRelay1 } from '@nostrify/nostrify'; import { getRelays } from '@/utils/outbox.ts'; import { seedZapSplits } from '@/utils/zap-split.ts'; +DittoExit.add(() => Storages.close()); + export class Storages { private static _db: Promise | undefined; - private static _database: DittoDatabase | undefined; + private static _database: Promise | undefined; private static _admin: Promise | undefined; private static _client: Promise | undefined; private static _pubsub: Promise | undefined; @@ -20,8 +23,12 @@ export class Storages { public static async database(): Promise { if (!this._database) { - this._database = DittoDB.create(Conf.databaseUrl, { poolSize: Conf.pg.poolSize }); - await DittoDB.migrate(this._database.kysely); + this._database = (async () => { + const db = DittoDB.create(Conf.databaseUrl, { poolSize: Conf.pg.poolSize }); + await db.waitReady; + await DittoDB.migrate(db.kysely); + return db; + })(); } return this._database; } @@ -35,7 +42,7 @@ export class Storages { public static async db(): Promise { if (!this._db) { this._db = (async () => { - const { kysely } = await this.database(); + const kysely = await this.kysely(); const store = new EventsDB({ kysely, pubkey: Conf.pubkey, timeout: Conf.db.timeouts.default }); await seedZapSplits(store); return store; @@ -118,4 +125,12 @@ export class Storages { } return this._search; } + + /** Close the database connection, if one has been opened. */ + public static async close(): Promise { + if (this._database) { + const { kysely } = await this._database; + await kysely.destroy(); + } + } } From c50c63f954c0fb6cb72ad5e4f53dd0d79d580b86 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Thu, 12 Sep 2024 13:10:36 -0500 Subject: [PATCH 47/82] pipeline: purifyEvent before passing it to storage --- src/pipeline.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/pipeline.ts b/src/pipeline.ts index afc7acdf..85d27964 100644 --- a/src/pipeline.ts +++ b/src/pipeline.ts @@ -5,6 +5,7 @@ import { LRUCache } from 'lru-cache'; import { z } from 'zod'; import { Conf } from '@/config.ts'; +import { DittoTables } from '@/db/DittoTables.ts'; import { DittoEvent } from '@/interfaces/DittoEvent.ts'; import { pipelineEventsCounter, policyEventsCounter } from '@/metrics.ts'; import { RelayError } from '@/RelayError.ts'; @@ -14,11 +15,11 @@ import { Storages } from '@/storages.ts'; import { eventAge, parseNip05, Time } from '@/utils.ts'; import { policyWorker } from '@/workers/policy.ts'; import { verifyEventWorker } from '@/workers/verify.ts'; +import { getAmount } from '@/utils/bolt11.ts'; import { nip05Cache } from '@/utils/nip05.ts'; +import { purifyEvent } from '@/utils/purify.ts'; import { updateStats } from '@/utils/stats.ts'; import { getTagSet } from '@/utils/tags.ts'; -import { getAmount } from '@/utils/bolt11.ts'; -import { DittoTables } from '@/db/DittoTables.ts'; const debug = Debug('ditto:pipeline'); @@ -55,7 +56,7 @@ async function handleEvent(event: DittoEvent, signal: AbortSignal): Promise Date: Thu, 12 Sep 2024 13:17:21 -0500 Subject: [PATCH 48/82] waitReady is not actually needed --- src/db/DittoDatabase.ts | 1 - src/db/adapters/DittoPglite.ts | 1 - src/db/adapters/DittoPostgres.ts | 1 - src/storages.ts | 1 - 4 files changed, 4 deletions(-) diff --git a/src/db/DittoDatabase.ts b/src/db/DittoDatabase.ts index 93c71a90..530d9391 100644 --- a/src/db/DittoDatabase.ts +++ b/src/db/DittoDatabase.ts @@ -6,7 +6,6 @@ export interface DittoDatabase { readonly kysely: Kysely; readonly poolSize: number; readonly availableConnections: number; - readonly waitReady: Promise; } export interface DittoDatabaseOpts { diff --git a/src/db/adapters/DittoPglite.ts b/src/db/adapters/DittoPglite.ts index 9b425c4b..3423cb31 100644 --- a/src/db/adapters/DittoPglite.ts +++ b/src/db/adapters/DittoPglite.ts @@ -21,7 +21,6 @@ export class DittoPglite { kysely, poolSize: 1, availableConnections: 1, - waitReady: pglite.waitReady, }; } } diff --git a/src/db/adapters/DittoPostgres.ts b/src/db/adapters/DittoPostgres.ts index 0300c3e0..f1a5bcc9 100644 --- a/src/db/adapters/DittoPostgres.ts +++ b/src/db/adapters/DittoPostgres.ts @@ -48,7 +48,6 @@ export class DittoPostgres { get availableConnections() { return pg.connections.idle; }, - waitReady: Promise.resolve(), }; } } diff --git a/src/storages.ts b/src/storages.ts index fb66c55e..15920444 100644 --- a/src/storages.ts +++ b/src/storages.ts @@ -25,7 +25,6 @@ export class Storages { if (!this._database) { this._database = (async () => { const db = DittoDB.create(Conf.databaseUrl, { poolSize: Conf.pg.poolSize }); - await db.waitReady; await DittoDB.migrate(db.kysely); return db; })(); From 83167623705848acd46c83b910a7f606714a6c53 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Thu, 12 Sep 2024 13:37:21 -0500 Subject: [PATCH 49/82] Remove this DittoExit stuff (since I'm not convinced it's needed) --- src/DittoExit.ts | 37 ------------------------------------- src/server.ts | 13 +------------ src/storages.ts | 3 --- 3 files changed, 1 insertion(+), 52 deletions(-) delete mode 100644 src/DittoExit.ts diff --git a/src/DittoExit.ts b/src/DittoExit.ts deleted file mode 100644 index 36201fc7..00000000 --- a/src/DittoExit.ts +++ /dev/null @@ -1,37 +0,0 @@ -import { Stickynotes } from '@soapbox/stickynotes'; - -/** - * Add cleanup tasks to this module, - * then they will automatically be called (and the program exited) after SIGINT. - */ -export class DittoExit { - private static tasks: Array<() => Promise> = []; - private static console = new Stickynotes('ditto:exit'); - - static { - Deno.addSignalListener('SIGINT', () => this.finish('SIGINT')); - Deno.addSignalListener('SIGTERM', () => this.finish('SIGTERM')); - Deno.addSignalListener('SIGHUP', () => this.finish('SIGHUP')); - Deno.addSignalListener('SIGQUIT', () => this.finish('SIGQUIT')); - Deno.addSignalListener('SIGABRT', () => this.finish('SIGABRT')); - } - - static add(task: () => Promise): void { - this.tasks.push(task); - this.console.debug(`Added cleanup task #${this.tasks.length}`); - } - - private static async cleanup(): Promise { - this.console.debug(`Running ${this.tasks.length} cleanup tasks...`); - await Promise.allSettled( - this.tasks.map((task) => task()), - ); - } - - private static async finish(signal: Deno.Signal): Promise { - this.console.debug(signal); - await this.cleanup(); - this.console.debug('Exiting gracefully.'); - Deno.exit(0); - } -} diff --git a/src/server.ts b/src/server.ts index bfa240c9..f7a33dc0 100644 --- a/src/server.ts +++ b/src/server.ts @@ -5,16 +5,5 @@ import '@/sentry.ts'; import '@/nostr-wasm.ts'; import app from '@/app.ts'; import { Conf } from '@/config.ts'; -import { DittoExit } from '@/DittoExit.ts'; -const ac = new AbortController(); -// deno-lint-ignore require-await -DittoExit.add(async () => ac.abort()); - -Deno.serve( - { - port: Conf.port, - signal: ac.signal, - }, - app.fetch, -); +Deno.serve({ port: Conf.port }, app.fetch); diff --git a/src/storages.ts b/src/storages.ts index 15920444..05e2c383 100644 --- a/src/storages.ts +++ b/src/storages.ts @@ -2,7 +2,6 @@ import { Conf } from '@/config.ts'; import { DittoDatabase } from '@/db/DittoDatabase.ts'; import { DittoDB } from '@/db/DittoDB.ts'; -import { DittoExit } from '@/DittoExit.ts'; import { AdminStore } from '@/storages/AdminStore.ts'; import { EventsDB } from '@/storages/EventsDB.ts'; import { SearchStore } from '@/storages/search-store.ts'; @@ -11,8 +10,6 @@ import { NPool, NRelay1 } from '@nostrify/nostrify'; import { getRelays } from '@/utils/outbox.ts'; import { seedZapSplits } from '@/utils/zap-split.ts'; -DittoExit.add(() => Storages.close()); - export class Storages { private static _db: Promise | undefined; private static _database: Promise | undefined; From a0fd702e09cc725c29d37133c098112f8bf921f4 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Thu, 12 Sep 2024 13:38:00 -0500 Subject: [PATCH 50/82] Revert DittoPglite --- src/db/adapters/DittoPglite.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/db/adapters/DittoPglite.ts b/src/db/adapters/DittoPglite.ts index 3423cb31..4ec7d8a5 100644 --- a/src/db/adapters/DittoPglite.ts +++ b/src/db/adapters/DittoPglite.ts @@ -8,11 +8,9 @@ import { KyselyLogger } from '@/db/KyselyLogger.ts'; export class DittoPglite { static create(databaseUrl: string): DittoDatabase { - const pglite = new PGlite(databaseUrl); - const kysely = new Kysely({ dialect: new PgliteDialect({ - database: pglite, + database: new PGlite(databaseUrl), }), log: KyselyLogger, }); From 1732b690220995bc237bbcf4ed84d6e053a4c5a1 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Thu, 12 Sep 2024 13:39:02 -0500 Subject: [PATCH 51/82] Remove unused Storages.close method --- src/storages.ts | 8 -------- 1 file changed, 8 deletions(-) diff --git a/src/storages.ts b/src/storages.ts index 05e2c383..cbafd5aa 100644 --- a/src/storages.ts +++ b/src/storages.ts @@ -121,12 +121,4 @@ export class Storages { } return this._search; } - - /** Close the database connection, if one has been opened. */ - public static async close(): Promise { - if (this._database) { - const { kysely } = await this._database; - await kysely.destroy(); - } - } } From b384fcf572f2378c02598275a1c9e70a5bce120b Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Thu, 12 Sep 2024 17:22:44 -0500 Subject: [PATCH 52/82] Make EventsDB extend NPostgres --- src/storages/EventsDB.ts | 46 ++++++++++++---------------------------- 1 file changed, 14 insertions(+), 32 deletions(-) diff --git a/src/storages/EventsDB.ts b/src/storages/EventsDB.ts index 148a30aa..bedc1cac 100644 --- a/src/storages/EventsDB.ts +++ b/src/storages/EventsDB.ts @@ -1,17 +1,7 @@ // deno-lint-ignore-file require-await -import { NPostgres } from '@nostrify/db'; -import { - NIP50, - NKinds, - NostrEvent, - NostrFilter, - NostrRelayCLOSED, - NostrRelayEOSE, - NostrRelayEVENT, - NSchema as n, - NStore, -} from '@nostrify/nostrify'; +import { NPostgres, NPostgresSchema } from '@nostrify/db'; +import { NIP50, NKinds, NostrEvent, NostrFilter, NSchema as n } from '@nostrify/nostrify'; import { Stickynotes } from '@soapbox/stickynotes'; import { Kysely } from 'kysely'; import { nip27 } from 'nostr-tools'; @@ -41,8 +31,7 @@ interface EventsDBOpts { } /** SQL database storage adapter for Nostr events. */ -class EventsDB implements NStore { - private store: NPostgres; +class EventsDB extends NPostgres { private console = new Stickynotes('ditto:db:events'); /** Conditions for when to index certain tags. */ @@ -63,7 +52,7 @@ class EventsDB implements NStore { }; constructor(private opts: EventsDBOpts) { - this.store = new NPostgres(opts.kysely, { + super(opts.kysely, { indexTags: EventsDB.indexTags, indexSearch: EventsDB.searchText, }); @@ -82,7 +71,7 @@ class EventsDB implements NStore { await this.deleteEventsAdmin(event); try { - await this.store.event(event, { ...opts, timeout: opts.timeout ?? this.opts.timeout }); + await super.event(event, { ...opts, timeout: opts.timeout ?? this.opts.timeout }); } catch (e) { if (e.message === 'Cannot add a deleted event') { throw new RelayError('blocked', 'event deleted by user'); @@ -155,12 +144,9 @@ class EventsDB implements NStore { } } - /** Stream events from the database. */ - req( - filters: NostrFilter[], - opts: { signal?: AbortSignal } = {}, - ): AsyncIterable { - return this.store.req(filters, opts); + protected getFilterQuery(trx: Kysely, filter: NostrFilter) { + const query = super.getFilterQuery(trx, filter); + return query; } /** Get events for filters from the database. */ @@ -185,32 +171,28 @@ class EventsDB implements NStore { } if (opts.signal?.aborted) return Promise.resolve([]); - if (!filters.length) return Promise.resolve([]); this.console.debug('REQ', JSON.stringify(filters)); - return this.store.query(filters, { ...opts, timeout: opts.timeout ?? this.opts.timeout }); + return super.query(filters, { ...opts, timeout: opts.timeout ?? this.opts.timeout }); } /** Delete events based on filters from the database. */ async remove(filters: NostrFilter[], opts: { signal?: AbortSignal; timeout?: number } = {}): Promise { - if (!filters.length) return Promise.resolve(); this.console.debug('DELETE', JSON.stringify(filters)); - - return this.store.remove(filters, { ...opts, timeout: opts.timeout ?? this.opts.timeout }); + return super.remove(filters, { ...opts, timeout: opts.timeout ?? this.opts.timeout }); } /** Get number of events that would be returned by filters. */ async count( filters: NostrFilter[], opts: { signal?: AbortSignal; timeout?: number } = {}, - ): Promise<{ count: number; approximate: boolean }> { + ): Promise<{ count: number; approximate: any }> { if (opts.signal?.aborted) return Promise.reject(abortError()); - if (!filters.length) return Promise.resolve({ count: 0, approximate: false }); this.console.debug('COUNT', JSON.stringify(filters)); - return this.store.count(filters, { ...opts, timeout: opts.timeout ?? this.opts.timeout }); + return super.count(filters, { ...opts, timeout: opts.timeout ?? this.opts.timeout }); } /** Return only the tags that should be indexed. */ @@ -317,8 +299,8 @@ class EventsDB implements NStore { return filters; } - async transaction(callback: (store: NPostgres, kysely: Kysely) => Promise): Promise { - return this.store.transaction((store, kysely) => callback(store, kysely as unknown as Kysely)); + async transaction(callback: (store: NPostgres, kysely: Kysely) => Promise): Promise { + return super.transaction((store, kysely) => callback(store, kysely as unknown as Kysely)); } } From 40c187680e8580c2cec374e8dea87ca94bd17152 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Fri, 13 Sep 2024 10:22:05 -0300 Subject: [PATCH 53/82] feat: create author_search table --- src/db/migrations/032_add_author_search.ts | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 src/db/migrations/032_add_author_search.ts diff --git a/src/db/migrations/032_add_author_search.ts b/src/db/migrations/032_add_author_search.ts new file mode 100644 index 00000000..d5a93c06 --- /dev/null +++ b/src/db/migrations/032_add_author_search.ts @@ -0,0 +1,18 @@ +import { Kysely, sql } from 'kysely'; + +export async function up(db: Kysely): Promise { + await db.schema + .createTable('author_search') + .addColumn('pubkey', 'char(64)', (col) => col.primaryKey()) + .addColumn('search', 'text', (col) => col.notNull()) + .ifNotExists() + .execute(); + + await sql`CREATE EXTENSION IF NOT EXISTS pg_trgm;`.execute(db); + await sql`CREATE INDEX author_search_search_idx ON author_search USING GIN (search gin_trgm_ops);`.execute(db); +} + +export async function down(db: Kysely): Promise { + await db.schema.dropIndex('author_search_search_idx').ifExists().execute(); + await db.schema.dropTable('author_search').execute(); +} From 8bc8712cf37d542258c9eecf4179d99e0ec8496d Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Fri, 13 Sep 2024 10:23:08 -0300 Subject: [PATCH 54/82] feat: create and add author_search interface to DittoTables --- src/db/DittoTables.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/db/DittoTables.ts b/src/db/DittoTables.ts index a62c485d..38f7249c 100644 --- a/src/db/DittoTables.ts +++ b/src/db/DittoTables.ts @@ -6,6 +6,7 @@ export interface DittoTables extends NPostgresSchema { event_stats: EventStatsRow; pubkey_domains: PubkeyDomainRow; event_zaps: EventZapRow; + author_search: AuthorSearch; } interface AuthorStatsRow { @@ -47,3 +48,8 @@ interface EventZapRow { amount_millisats: number; comment: string; } + +interface AuthorSearch { + pubkey: string; + search: string; +} From b5aefdd93e00d2921005f763013dccbbd7dde759 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Fri, 13 Sep 2024 10:24:33 -0300 Subject: [PATCH 55/82] feat: add pg_trgm extension in PGlite constructor --- src/db/adapters/DittoPglite.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/db/adapters/DittoPglite.ts b/src/db/adapters/DittoPglite.ts index 4ec7d8a5..0e93075d 100644 --- a/src/db/adapters/DittoPglite.ts +++ b/src/db/adapters/DittoPglite.ts @@ -1,4 +1,5 @@ import { PGlite } from '@electric-sql/pglite'; +import { pg_trgm } from '@electric-sql/pglite/contrib/pg_trgm'; import { PgliteDialect } from '@soapbox/kysely-pglite'; import { Kysely } from 'kysely'; @@ -10,7 +11,7 @@ export class DittoPglite { static create(databaseUrl: string): DittoDatabase { const kysely = new Kysely({ dialect: new PgliteDialect({ - database: new PGlite(databaseUrl), + database: new PGlite(databaseUrl, { extensions: { pg_trgm } }), }), log: KyselyLogger, }); From c03ea07dcb253e6ff1b59c403133ee72cea2f7bf Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Fri, 13 Sep 2024 10:27:12 -0300 Subject: [PATCH 56/82] feat: create getPubkeysBySearch() function and use it inside searchEvents() function --- src/controllers/api/search.ts | 30 ++++++++++++++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/src/controllers/api/search.ts b/src/controllers/api/search.ts index 9bddc336..19e72cac 100644 --- a/src/controllers/api/search.ts +++ b/src/controllers/api/search.ts @@ -1,8 +1,10 @@ import { NostrEvent, NostrFilter, NSchema as n } from '@nostrify/nostrify'; import { nip19 } from 'nostr-tools'; +import { Kysely, sql } from 'kysely'; import { z } from 'zod'; import { AppController } from '@/app.ts'; +import { DittoTables } from '@/db/DittoTables.ts'; import { booleanParamSchema } from '@/schema.ts'; import { Storages } from '@/storages.ts'; import { hydrateEvents } from '@/storages/hydrate.ts'; @@ -89,9 +91,21 @@ async function searchEvents({ q, type, limit, account_id }: SearchQuery, signal: filter.authors = [account_id]; } + const filter2: NostrFilter = { + kinds: [0], + limit, + }; + if (type === 'accounts') { + const kysely = await Storages.kysely(); + + const pubkeys = await getPubkeysBySearch(kysely, { q, limit }); + + filter2.authors = pubkeys; // if pubkeys is empty the filter 2 will be discarded + } + const store = await Storages.search(); - return store.query([filter], { signal }) + return store.query([filter, filter2], { signal }) .then((events) => hydrateEvents({ events, store, signal })); } @@ -170,4 +184,16 @@ async function getLookupFilters({ q, type, resolve }: SearchQuery, signal: Abort return []; } -export { searchController }; +/** Get pubkeys whose name and NIP-05 is similar to 'q' */ +async function getPubkeysBySearch(kysely: Kysely, { q, limit }: Pick) { + const pubkeys = (await sql` + SELECT *, word_similarity(${q}, search) AS sml + FROM author_search + WHERE ${q} % search + ORDER BY sml DESC, search LIMIT ${limit} + `.execute(kysely)).rows.map((row) => (row as { pubkey: string }).pubkey); + + return pubkeys; +} + +export { getPubkeysBySearch, searchController }; From 9d2667679fdac0adc956b4927ddd8417e4f08500 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Fri, 13 Sep 2024 10:28:54 -0300 Subject: [PATCH 57/82] feat(pipeline.ts): create handleAuthorSearch() function --- src/pipeline.ts | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/src/pipeline.ts b/src/pipeline.ts index 88e5f29b..9d1a8038 100644 --- a/src/pipeline.ts +++ b/src/pipeline.ts @@ -57,6 +57,7 @@ async function handleEvent(event: DittoEvent, signal: AbortSignal): Promise, event: NostrEvent) { } } +async function handleAuthorSearch(kysely: Kysely, event: NostrEvent) { + if (event.kind !== 0) return; + const { name, nip05 } = n.json().pipe(n.metadata()).catch({}).parse(event.content); + const search = [name, nip05].filter(Boolean).join(' ').trim(); + + try { + await kysely.insertInto('author_search').values({ + pubkey: event.pubkey, + search, + }).onConflict( + (oc) => + oc.column('pubkey') + .doUpdateSet({ search }), + ) + .execute(); + } catch { + // do nothing + } +} + export { handleEvent, handleZaps }; From e1cd1777e3d76b8f3b67d8d415215dc7e70ba424 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Fri, 13 Sep 2024 10:30:10 -0300 Subject: [PATCH 58/82] test: add author_search table in createTestDB to drop it after use --- src/test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/test.ts b/src/test.ts index 45946f00..00bf4354 100644 --- a/src/test.ts +++ b/src/test.ts @@ -63,6 +63,7 @@ export async function createTestDB() { 'pubkey_domains', 'nostr_events', 'event_zaps', + 'author_search', ] ) { await kysely.schema.dropTable(table).ifExists().cascade().execute(); From a6f1098bc6fc52dee99de61983399961af5c1aa8 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Fri, 13 Sep 2024 10:32:05 -0300 Subject: [PATCH 59/82] test: getPubkeysBySearch() function --- src/controllers/api/search.test.ts | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 src/controllers/api/search.test.ts diff --git a/src/controllers/api/search.test.ts b/src/controllers/api/search.test.ts new file mode 100644 index 00000000..2c5e91bd --- /dev/null +++ b/src/controllers/api/search.test.ts @@ -0,0 +1,21 @@ +import { assertEquals } from '@std/assert'; + +import { createTestDB } from '@/test.ts'; +import { getPubkeysBySearch } from '@/controllers/api/search.ts'; + +Deno.test('fuzzy search works', async () => { + await using db = await createTestDB(); + + await db.kysely.insertInto('author_search').values({ + pubkey: '47259076c85f9240e852420d7213c95e95102f1de929fb60f33a2c32570c98c4', + search: 'patrickReiis patrickdosreis.com', + }).execute(); + + assertEquals(await getPubkeysBySearch(db.kysely, { q: 'pat rick', limit: 1 }), []); + assertEquals(await getPubkeysBySearch(db.kysely, { q: 'patrick dos reis', limit: 1 }), [ + '47259076c85f9240e852420d7213c95e95102f1de929fb60f33a2c32570c98c4', + ]); + assertEquals(await getPubkeysBySearch(db.kysely, { q: 'dosreis.com', limit: 1 }), [ + '47259076c85f9240e852420d7213c95e95102f1de929fb60f33a2c32570c98c4', + ]); +}); From 24d909fd28383fda2f10934e7246483fe1be52ff Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Fri, 13 Sep 2024 10:34:15 -0300 Subject: [PATCH 60/82] feat: create script to populate author_search table --- deno.json | 3 ++- scripts/db-populate-search.ts | 35 +++++++++++++++++++++++++++++++++++ 2 files changed, 37 insertions(+), 1 deletion(-) create mode 100644 scripts/db-populate-search.ts diff --git a/deno.json b/deno.json index 699ab620..0070b142 100644 --- a/deno.json +++ b/deno.json @@ -18,7 +18,8 @@ "stats:recompute": "deno run -A scripts/stats-recompute.ts", "soapbox": "curl -O https://dl.soapbox.pub/main/soapbox.zip && mkdir -p public && mv soapbox.zip public/ && cd public/ && unzip -o soapbox.zip && rm soapbox.zip", "trends": "deno run -A scripts/trends.ts", - "clean:deps": "deno cache --reload src/app.ts" + "clean:deps": "deno cache --reload src/app.ts", + "db:populate-search": "deno run -A scripts/db-populate-search.ts" }, "unstable": ["cron", "ffi", "kv", "worker-options"], "exclude": ["./public"], diff --git a/scripts/db-populate-search.ts b/scripts/db-populate-search.ts new file mode 100644 index 00000000..9c698c8f --- /dev/null +++ b/scripts/db-populate-search.ts @@ -0,0 +1,35 @@ +import { NSchema as n } from '@nostrify/nostrify'; +import { Storages } from '@/storages.ts'; +import { DittoTables } from '@/db/DittoTables.ts'; + +const kysely = await Storages.kysely(); +const stream = kysely + .selectFrom('nostr_events') + .select(['pubkey', 'content']) + .where('kind', '=', 0) + .stream(); + +const values: DittoTables['author_search'][] = []; + +for await (const author of stream) { + const { name, nip05 } = n.json().pipe(n.metadata()).catch({}).parse(author.content); + const search = [name, nip05].filter(Boolean).join(' ').trim(); + + values.push({ + pubkey: author.pubkey, + search, + }); +} + +try { + await kysely.insertInto('author_search').values(values).onConflict( + (oc) => + oc.column('pubkey') + .doUpdateSet((eb) => ({ search: eb.ref('excluded.search') })), + ) + .execute(); +} catch { + // do nothing +} + +Deno.exit(); From 935cc7c5a574ba9bf5cb148a14f4f0db3a932377 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Fri, 13 Sep 2024 14:33:34 -0300 Subject: [PATCH 61/82] refactor: remove NIP-50 search if looking for accounts, use same filter --- src/controllers/api/search.ts | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/src/controllers/api/search.ts b/src/controllers/api/search.ts index 19e72cac..1a1bd867 100644 --- a/src/controllers/api/search.ts +++ b/src/controllers/api/search.ts @@ -91,21 +91,18 @@ async function searchEvents({ q, type, limit, account_id }: SearchQuery, signal: filter.authors = [account_id]; } - const filter2: NostrFilter = { - kinds: [0], - limit, - }; if (type === 'accounts') { const kysely = await Storages.kysely(); const pubkeys = await getPubkeysBySearch(kysely, { q, limit }); - filter2.authors = pubkeys; // if pubkeys is empty the filter 2 will be discarded + filter.authors = pubkeys; + filter.search = undefined; } const store = await Storages.search(); - return store.query([filter, filter2], { signal }) + return store.query([filter], { signal }) .then((events) => hydrateEvents({ events, store, signal })); } From be76197e3ac92ceae17a50db907fda4b4ee91a76 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Fri, 13 Sep 2024 15:08:36 -0300 Subject: [PATCH 62/82] refactor: remove handleAuthorSearch() function and put its logic inside parseMetadata() function --- src/pipeline.ts | 44 +++++++++++++++++++++----------------------- 1 file changed, 21 insertions(+), 23 deletions(-) diff --git a/src/pipeline.ts b/src/pipeline.ts index b208e9e2..aeba0150 100644 --- a/src/pipeline.ts +++ b/src/pipeline.ts @@ -58,7 +58,6 @@ async function handleEvent(event: DittoEvent, signal: AbortSignal): Promise + oc.column('pubkey') + .doUpdateSet({ search }), + ) + .execute(); + } catch { + // do nothing + } + if (!nip05) return; // Fetch nip05. @@ -148,7 +167,6 @@ async function parseMetadata(event: NostrEvent, signal: AbortSignal): Promise, event: NostrEvent) { } } -async function handleAuthorSearch(kysely: Kysely, event: NostrEvent) { - if (event.kind !== 0) return; - const { name, nip05 } = n.json().pipe(n.metadata()).catch({}).parse(event.content); - const search = [name, nip05].filter(Boolean).join(' ').trim(); - - try { - await kysely.insertInto('author_search').values({ - pubkey: event.pubkey, - search, - }).onConflict( - (oc) => - oc.column('pubkey') - .doUpdateSet({ search }), - ) - .execute(); - } catch { - // do nothing - } -} - export { handleEvent, handleZaps }; From 69c21581310a720aed184fcb7f673282d7393d9f Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Fri, 13 Sep 2024 16:19:38 -0300 Subject: [PATCH 63/82] refactor: return ordered accounts by similarity relevance in searchEvents() function --- src/controllers/api/search.ts | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/src/controllers/api/search.ts b/src/controllers/api/search.ts index 1a1bd867..2555109b 100644 --- a/src/controllers/api/search.ts +++ b/src/controllers/api/search.ts @@ -91,19 +91,35 @@ async function searchEvents({ q, type, limit, account_id }: SearchQuery, signal: filter.authors = [account_id]; } + const pubkeys: string[] = []; if (type === 'accounts') { const kysely = await Storages.kysely(); - const pubkeys = await getPubkeysBySearch(kysely, { q, limit }); + pubkeys.push(...(await getPubkeysBySearch(kysely, { q, limit }))); + + if (!filter?.authors) filter.authors = pubkeys; + else filter.authors.push(...pubkeys); - filter.authors = pubkeys; filter.search = undefined; } const store = await Storages.search(); - return store.query([filter], { signal }) + const events = await store.query([filter], { signal }) .then((events) => hydrateEvents({ events, store, signal })); + + if (type !== 'accounts') return events; + + const orderedEvents: NostrEvent[] = events.map((event, index) => { + const pubkey = pubkeys[index]; + + const orderedEvent = events.find((e) => e.pubkey === pubkey); + if (orderedEvent) return orderedEvent; + + return event; + }); + + return orderedEvents; } /** Get event kinds to search from `type` query param. */ From 6387ee440cc9d7c773088f158c45cc17cab93a3c Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Fri, 13 Sep 2024 16:36:01 -0300 Subject: [PATCH 64/82] feat: return multiple accounts in searchController --- src/controllers/api/search.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/controllers/api/search.ts b/src/controllers/api/search.ts index 2555109b..c778bc4f 100644 --- a/src/controllers/api/search.ts +++ b/src/controllers/api/search.ts @@ -49,9 +49,8 @@ const searchController: AppController = async (c) => { if (event) { events = [event]; - } else { - events = await searchEvents(result.data, signal); } + events.push(...(await searchEvents(result.data, signal))); const viewerPubkey = await c.get('signer')?.getPublicKey(); From b3e56320a04ab866b9aeeb782e794d6252be1f49 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Fri, 13 Sep 2024 18:41:20 -0300 Subject: [PATCH 65/82] feat(accountSearchController): return accounts in autocomplete form --- src/controllers/api/accounts.ts | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/src/controllers/api/accounts.ts b/src/controllers/api/accounts.ts index 9c635565..a8aa5d55 100644 --- a/src/controllers/api/accounts.ts +++ b/src/controllers/api/accounts.ts @@ -6,6 +6,7 @@ import { type AppController } from '@/app.ts'; import { Conf } from '@/config.ts'; import { getAuthor, getFollowedPubkeys } from '@/queries.ts'; import { booleanParamSchema, fileSchema } from '@/schema.ts'; +import { getPubkeysBySearch } from '@/controllers/api/search.ts'; import { Storages } from '@/storages.ts'; import { uploadFile } from '@/utils/upload.ts'; import { nostrNow } from '@/utils.ts'; @@ -115,6 +116,7 @@ const accountSearchQuerySchema = z.object({ const accountSearchController: AppController = async (c) => { const { signal } = c.req.raw; const { limit } = c.get('pagination'); + const kysely = await Storages.kysely(); const result = accountSearchQuerySchema.safeParse(c.req.query()); @@ -133,9 +135,22 @@ const accountSearchController: AppController = async (c) => { return c.json(pubkey ? [await accountFromPubkey(pubkey)] : []); } - const events = event ? [event] : await store.query([{ kinds: [0], search: query, limit }], { signal }); + const pubkeys = await getPubkeysBySearch(kysely, { q: query, limit }); - const accounts = await hydrateEvents({ events, store, signal }).then( + const events = event ? [event] : await store.query([{ kinds: [0], authors: pubkeys, limit }], { + signal, + }); + + const orderedEvents = events.map((event, index) => { + const pubkey = pubkeys[index]; + + const orderedEvent = events.find((e) => e.pubkey === pubkey); + if (orderedEvent) return orderedEvent; + + return event; + }); + + const accounts = await hydrateEvents({ events: orderedEvents, store, signal }).then( (events) => Promise.all( events.map((event) => renderAccount(event)), From 197b2c8c8b3b62e005134e405a430c55a0bbc0c5 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Fri, 13 Sep 2024 18:49:20 -0300 Subject: [PATCH 66/82] refactor(populate search script): use store.req instead of streaming --- scripts/db-populate-search.ts | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/scripts/db-populate-search.ts b/scripts/db-populate-search.ts index 9c698c8f..beec2d52 100644 --- a/scripts/db-populate-search.ts +++ b/scripts/db-populate-search.ts @@ -2,23 +2,23 @@ import { NSchema as n } from '@nostrify/nostrify'; import { Storages } from '@/storages.ts'; import { DittoTables } from '@/db/DittoTables.ts'; +const store = await Storages.db(); const kysely = await Storages.kysely(); -const stream = kysely - .selectFrom('nostr_events') - .select(['pubkey', 'content']) - .where('kind', '=', 0) - .stream(); const values: DittoTables['author_search'][] = []; -for await (const author of stream) { - const { name, nip05 } = n.json().pipe(n.metadata()).catch({}).parse(author.content); - const search = [name, nip05].filter(Boolean).join(' ').trim(); +for await (const msg of store.req([{ kinds: [0] }])) { + if (msg[0] === 'EVENT') { + const { pubkey, content } = msg[2]; - values.push({ - pubkey: author.pubkey, - search, - }); + const { name, nip05 } = n.json().pipe(n.metadata()).catch({}).parse(content); + const search = [name, nip05].filter(Boolean).join(' ').trim(); + + values.push({ + pubkey: pubkey, + search, + }); + } } try { From f99ea7c33fa4cf385513a487df164b48ea1d3f71 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Fri, 13 Sep 2024 18:57:47 -0300 Subject: [PATCH 67/82] refactor(getPubkeysBySearch): cast as string --- src/controllers/api/search.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/controllers/api/search.ts b/src/controllers/api/search.ts index c778bc4f..d62c08e8 100644 --- a/src/controllers/api/search.ts +++ b/src/controllers/api/search.ts @@ -198,12 +198,12 @@ async function getLookupFilters({ q, type, resolve }: SearchQuery, signal: Abort /** Get pubkeys whose name and NIP-05 is similar to 'q' */ async function getPubkeysBySearch(kysely: Kysely, { q, limit }: Pick) { - const pubkeys = (await sql` + const pubkeys = (await sql<{ pubkey: string }>` SELECT *, word_similarity(${q}, search) AS sml FROM author_search WHERE ${q} % search ORDER BY sml DESC, search LIMIT ${limit} - `.execute(kysely)).rows.map((row) => (row as { pubkey: string }).pubkey); + `.execute(kysely)).rows.map(({ pubkey }) => pubkey); return pubkeys; } From c24d11c6f3055cc20a5b55abd1281bf6ce28020c Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 15 Sep 2024 12:40:58 -0500 Subject: [PATCH 68/82] Support NIP-50 language extension --- deno.json | 1 + deno.lock | 66 +++++++++++++++++++++++++-- src/db/DittoTables.ts | 7 +++ src/db/migrations/032_add_language.ts | 11 +++++ src/pipeline.ts | 27 ++++++++++- src/storages/EventsDB.test.ts | 17 +++++++ src/storages/EventsDB.ts | 63 +++++++++++++------------ 7 files changed, 156 insertions(+), 36 deletions(-) create mode 100644 src/db/migrations/032_add_language.ts diff --git a/deno.json b/deno.json index 699ab620..8edf0e53 100644 --- a/deno.json +++ b/deno.json @@ -59,6 +59,7 @@ "isomorphic-dompurify": "npm:isomorphic-dompurify@^2.11.0", "kysely": "npm:kysely@^0.27.4", "kysely-postgres-js": "npm:kysely-postgres-js@2.0.0", + "lande": "npm:lande@^1.0.10", "light-bolt11-decoder": "npm:light-bolt11-decoder", "linkify-plugin-hashtag": "npm:linkify-plugin-hashtag@^4.1.1", "linkify-string": "npm:linkify-string@^4.1.1", diff --git a/deno.lock b/deno.lock index a275accb..3a25d8d3 100644 --- a/deno.lock +++ b/deno.lock @@ -29,13 +29,16 @@ "jsr:@soapbox/kysely-pglite@^0.0.1": "jsr:@soapbox/kysely-pglite@0.0.1", "jsr:@soapbox/stickynotes@^0.4.0": "jsr:@soapbox/stickynotes@0.4.0", "jsr:@std/assert@^0.213.1": "jsr:@std/assert@0.213.1", + "jsr:@std/assert@^0.223.0": "jsr:@std/assert@0.223.0", "jsr:@std/assert@^0.224.0": "jsr:@std/assert@0.224.0", "jsr:@std/assert@^0.225.1": "jsr:@std/assert@0.225.3", + "jsr:@std/bytes@^0.223.0": "jsr:@std/bytes@0.223.0", "jsr:@std/bytes@^0.224.0": "jsr:@std/bytes@0.224.0", "jsr:@std/bytes@^1.0.0-rc.3": "jsr:@std/bytes@1.0.0", "jsr:@std/bytes@^1.0.1-rc.3": "jsr:@std/bytes@1.0.2", "jsr:@std/bytes@^1.0.2": "jsr:@std/bytes@1.0.2", "jsr:@std/bytes@^1.0.2-rc.3": "jsr:@std/bytes@1.0.2", + "jsr:@std/cli@^0.223.0": "jsr:@std/cli@0.223.0", "jsr:@std/crypto@^0.224.0": "jsr:@std/crypto@0.224.0", "jsr:@std/dotenv@^0.224.0": "jsr:@std/dotenv@0.224.2", "jsr:@std/encoding@0.213.1": "jsr:@std/encoding@0.213.1", @@ -45,14 +48,17 @@ "jsr:@std/fs@0.213.1": "jsr:@std/fs@0.213.1", "jsr:@std/fs@^0.229.3": "jsr:@std/fs@0.229.3", "jsr:@std/internal@^1.0.0": "jsr:@std/internal@1.0.1", + "jsr:@std/io@^0.223.0": "jsr:@std/io@0.223.0", "jsr:@std/io@^0.224": "jsr:@std/io@0.224.7", "jsr:@std/json@^0.223.0": "jsr:@std/json@0.223.0", "jsr:@std/media-types@^0.224.1": "jsr:@std/media-types@0.224.1", "jsr:@std/path@0.213.1": "jsr:@std/path@0.213.1", + "jsr:@std/path@1.0.0-rc.1": "jsr:@std/path@1.0.0-rc.1", "jsr:@std/path@^0.213.1": "jsr:@std/path@0.213.1", "jsr:@std/streams@^0.223.0": "jsr:@std/streams@0.223.0", "npm:@isaacs/ttlcache@^1.4.1": "npm:@isaacs/ttlcache@1.4.1", "npm:@noble/hashes@^1.4.0": "npm:@noble/hashes@1.4.0", + "npm:@noble/secp256k1@^2.0.0": "npm:@noble/secp256k1@2.1.0", "npm:@scure/base@^1.1.6": "npm:@scure/base@1.1.6", "npm:@scure/bip32@^1.4.0": "npm:@scure/bip32@1.4.0", "npm:@scure/bip39@^1.3.0": "npm:@scure/bip39@1.3.0", @@ -72,6 +78,7 @@ "npm:kysely@^0.27.2": "npm:kysely@0.27.4", "npm:kysely@^0.27.3": "npm:kysely@0.27.4", "npm:kysely@^0.27.4": "npm:kysely@0.27.4", + "npm:lande@^1.0.10": "npm:lande@1.0.10", "npm:light-bolt11-decoder": "npm:light-bolt11-decoder@3.1.1", "npm:linkify-plugin-hashtag@^4.1.1": "npm:linkify-plugin-hashtag@4.1.3_linkifyjs@4.1.3", "npm:linkify-string@^4.1.1": "npm:linkify-string@4.1.3_linkifyjs@4.1.3", @@ -88,6 +95,7 @@ "npm:postgres@3.4.4": "npm:postgres@3.4.4", "npm:prom-client@^15.1.2": "npm:prom-client@15.1.2", "npm:tldts@^6.0.14": "npm:tldts@6.1.18", + "npm:tseep@^1.2.1": "npm:tseep@1.2.1", "npm:type-fest@^4.3.0": "npm:type-fest@4.18.2", "npm:unfurl.js@^6.4.0": "npm:unfurl.js@6.4.0", "npm:websocket-ts@^2.1.5": "npm:websocket-ts@2.1.5", @@ -303,6 +311,9 @@ "@std/assert@0.213.1": { "integrity": "24c28178b30c8e0782c18e8e94ea72b16282207569cdd10ffb9d1d26f2edebfe" }, + "@std/assert@0.223.0": { + "integrity": "eb8d6d879d76e1cc431205bd346ed4d88dc051c6366365b1af47034b0670be24" + }, "@std/assert@0.224.0": { "integrity": "8643233ec7aec38a940a8264a6e3eed9bfa44e7a71cc6b3c8874213ff401967f" }, @@ -312,6 +323,9 @@ "jsr:@std/internal@^1.0.0" ] }, + "@std/bytes@0.223.0": { + "integrity": "84b75052cd8680942c397c2631318772b295019098f40aac5c36cead4cba51a8" + }, "@std/bytes@0.224.0": { "integrity": "a2250e1d0eb7d1c5a426f21267ab9bdeac2447fa87a3d0d1a467d3f7a6058e49" }, @@ -321,6 +335,12 @@ "@std/bytes@1.0.2": { "integrity": "fbdee322bbd8c599a6af186a1603b3355e59a5fb1baa139f8f4c3c9a1b3e3d57" }, + "@std/cli@0.223.0": { + "integrity": "2feb7970f2028904c3edc22ea916ce9538113dfc170844f3eae03578c333c356", + "dependencies": [ + "jsr:@std/assert@^0.223.0" + ] + }, "@std/crypto@0.224.0": { "integrity": "154ef3ff08ef535562ef1a718718c5b2c5fc3808f0f9100daad69e829bfcdf2d", "dependencies": [ @@ -351,7 +371,10 @@ ] }, "@std/fs@0.229.3": { - "integrity": "783bca21f24da92e04c3893c9e79653227ab016c48e96b3078377ebd5222e6eb" + "integrity": "783bca21f24da92e04c3893c9e79653227ab016c48e96b3078377ebd5222e6eb", + "dependencies": [ + "jsr:@std/path@1.0.0-rc.1" + ] }, "@std/internal@1.0.0": { "integrity": "ac6a6dfebf838582c4b4f61a6907374e27e05bedb6ce276e0f1608fe84e7cd9a" @@ -359,6 +382,13 @@ "@std/internal@1.0.1": { "integrity": "6f8c7544d06a11dd256c8d6ba54b11ed870aac6c5aeafff499892662c57673e6" }, + "@std/io@0.223.0": { + "integrity": "2d8c3c2ab3a515619b90da2c6ff5ea7b75a94383259ef4d02116b228393f84f1", + "dependencies": [ + "jsr:@std/assert@^0.223.0", + "jsr:@std/bytes@^0.223.0" + ] + }, "@std/io@0.224.0": { "integrity": "0aff885d21d829c050b8a08b1d71b54aed5841aecf227f8d77e99ec529a11e8e", "dependencies": [ @@ -396,7 +426,10 @@ ] }, "@std/json@0.223.0": { - "integrity": "9a4a255931dd0397924c6b10bb6a72fe3e28ddd876b981ada2e3b8dd0764163f" + "integrity": "9a4a255931dd0397924c6b10bb6a72fe3e28ddd876b981ada2e3b8dd0764163f", + "dependencies": [ + "jsr:@std/streams@^0.223.0" + ] }, "@std/media-types@0.224.1": { "integrity": "9e69a5daed37c5b5c6d3ce4731dc191f80e67f79bed392b0957d1d03b87f11e1" @@ -407,8 +440,16 @@ "jsr:@std/assert@^0.213.1" ] }, + "@std/path@1.0.0-rc.1": { + "integrity": "b8c00ae2f19106a6bb7cbf1ab9be52aa70de1605daeb2dbdc4f87a7cbaf10ff6" + }, "@std/streams@0.223.0": { - "integrity": "d6b28e498ced3960b04dc5d251f2dcfc1df244b5ec5a48dc23a8f9b490be3b99" + "integrity": "d6b28e498ced3960b04dc5d251f2dcfc1df244b5ec5a48dc23a8f9b490be3b99", + "dependencies": [ + "jsr:@std/assert@^0.223.0", + "jsr:@std/bytes@^0.223.0", + "jsr:@std/io@^0.223.0" + ] } }, "npm": { @@ -454,6 +495,10 @@ "integrity": "sha512-V1JJ1WTRUqHHrOSh597hURcMqVKVGL/ea3kv0gSnEdsEZ0/+VyPghM1lMNGc00z7CIQorSvbKpuJkxvuHbvdbg==", "dependencies": {} }, + "@noble/secp256k1@2.1.0": { + "integrity": "sha512-XLEQQNdablO0XZOIniFQimiXsZDNwaYgL96dZwC54Q30imSbAOFf3NKtepc+cXyuZf5Q1HCgbqgZ2UFFuHVcEw==", + "dependencies": {} + }, "@opentelemetry/api@1.9.0": { "integrity": "sha512-3giAOQvZiH5F9bMlMiv8+GSPMeqg0dbaeo58/0SlA9sxSqZhnUtxzX9/2FzyhS9sWQf5S0GJE0AKBrFqjpeYcg==", "dependencies": {} @@ -864,6 +909,12 @@ "integrity": "sha512-dyNKv2KRvYOQPLCAOCjjQuCk4YFd33BvGdf/o5bC7FiW+BB6snA81Zt+2wT9QDFzKqxKa5rrOmvlK/anehCcgA==", "dependencies": {} }, + "lande@1.0.10": { + "integrity": "sha512-yT52DQh+UV2pEp08jOYrA4drDv0DbjpiRyZYgl25ak9G2cVR2AimzrqkYQWrD9a7Ud+qkAcaiDDoNH9DXfHPmw==", + "dependencies": { + "toygrad": "toygrad@2.6.0" + } + }, "light-bolt11-decoder@3.1.1": { "integrity": "sha512-sLg/KCwYkgsHWkefWd6KqpCHrLFWWaXTOX3cf6yD2hAzL0SLpX+lFcaFK2spkjbgzG6hhijKfORDc9WoUHwX0A==", "dependencies": { @@ -1213,6 +1264,10 @@ "url-parse": "url-parse@1.5.10" } }, + "toygrad@2.6.0": { + "integrity": "sha512-g4zBmlSbvzOE5FOILxYkAybTSxijKLkj1WoNqVGnbMcWDyj4wWQ+eYSr3ik7XOpIgMq/7eBcPRTJX3DM2E0YMg==", + "dependencies": {} + }, "tr46@0.0.3": { "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==", "dependencies": {} @@ -1223,6 +1278,10 @@ "punycode": "punycode@2.3.1" } }, + "tseep@1.2.1": { + "integrity": "sha512-VFnsNcPGC4qFJ1nxbIPSjTmtRZOhlqLmtwRqtLVos8mbRHki8HO9cy9Z1e89EiWyxFmq6LBviI9TQjijxw/mEw==", + "dependencies": {} + }, "type-fest@3.13.1": { "integrity": "sha512-tLq3bSNx+xSpwvAJnzrK0Ep5CLNWjvFTOp71URMaAEWBfRb9nnJiBoUe0tF8bI4ZFO3omgBR6NvnbzVUT3Ly4g==", "dependencies": {} @@ -1963,6 +2022,7 @@ "npm:isomorphic-dompurify@^2.11.0", "npm:kysely-postgres-js@2.0.0", "npm:kysely@^0.27.4", + "npm:lande@^1.0.10", "npm:light-bolt11-decoder", "npm:linkify-plugin-hashtag@^4.1.1", "npm:linkify-string@^4.1.1", diff --git a/src/db/DittoTables.ts b/src/db/DittoTables.ts index a62c485d..48cb06cb 100644 --- a/src/db/DittoTables.ts +++ b/src/db/DittoTables.ts @@ -1,6 +1,9 @@ +import { Nullable } from 'kysely'; + import { NPostgresSchema } from '@nostrify/db'; export interface DittoTables extends NPostgresSchema { + nostr_events: NostrEventsRow; nip46_tokens: NIP46TokenRow; author_stats: AuthorStatsRow; event_stats: EventStatsRow; @@ -8,6 +11,10 @@ export interface DittoTables extends NPostgresSchema { event_zaps: EventZapRow; } +type NostrEventsRow = NPostgresSchema['nostr_events'] & { + language: Nullable; +}; + interface AuthorStatsRow { pubkey: string; followers_count: number; diff --git a/src/db/migrations/032_add_language.ts b/src/db/migrations/032_add_language.ts new file mode 100644 index 00000000..a0f828fe --- /dev/null +++ b/src/db/migrations/032_add_language.ts @@ -0,0 +1,11 @@ +import { Kysely } from 'kysely'; + +export async function up(db: Kysely): Promise { + await db.schema.alterTable('nostr_events').addColumn('language', 'char(2)').execute(); + await db.schema.createIndex('nostr_events_language_idx').on('nostr_events').column('language').execute(); +} + +export async function down(db: Kysely): Promise { + await db.schema.alterTable('nostr_events').dropColumn('language').execute(); + await db.schema.dropIndex('nostr_events_language_idx').execute(); +} diff --git a/src/pipeline.ts b/src/pipeline.ts index 85d27964..cc4975cd 100644 --- a/src/pipeline.ts +++ b/src/pipeline.ts @@ -1,6 +1,8 @@ import { NKinds, NostrEvent, NSchema as n } from '@nostrify/nostrify'; import Debug from '@soapbox/stickynotes/debug'; +import ISO6391 from 'iso-639-1'; import { Kysely, sql } from 'kysely'; +import lande from 'lande'; import { LRUCache } from 'lru-cache'; import { z } from 'zod'; @@ -55,10 +57,11 @@ async function handleEvent(event: DittoEvent, signal: AbortSignal): Promise { + const [topResult] = lande(event.content); + + if (topResult) { + const [iso6393, confidence] = topResult; + const locale = new Intl.Locale(iso6393); + + if (confidence >= 0.95 && ISO6391.validate(locale.language)) { + const kysely = await Storages.kysely(); + try { + await kysely.updateTable('nostr_events') + .set('language', locale.language) + .where('id', '=', event.id) + .execute(); + } catch { + // do nothing + } + } + } +} + /** Determine if the event is being received in a timely manner. */ function isFresh(event: NostrEvent): boolean { return eventAge(event) < Time.seconds(10); diff --git a/src/storages/EventsDB.test.ts b/src/storages/EventsDB.test.ts index 7a5f7b93..b24032aa 100644 --- a/src/storages/EventsDB.test.ts +++ b/src/storages/EventsDB.test.ts @@ -54,6 +54,23 @@ Deno.test('query events with domain search filter', async () => { assertEquals(await store.query([{ kinds: [1], search: 'domain:example.com' }]), []); }); +Deno.test('query events with language search filter', async () => { + await using db = await createTestDB(); + const { store, kysely } = db; + + const en = genEvent({ kind: 1, content: 'hello world!' }); + const es = genEvent({ kind: 1, content: 'hola mundo!' }); + + await store.event(en); + await store.event(es); + + await kysely.updateTable('nostr_events').set('language', 'en').where('id', '=', en.id).execute(); + await kysely.updateTable('nostr_events').set('language', 'es').where('id', '=', es.id).execute(); + + assertEquals(await store.query([{ search: 'language:en' }]), [en]); + assertEquals(await store.query([{ search: 'language:es' }]), [es]); +}); + Deno.test('delete events', async () => { await using db = await createTestDB(); const { store } = db; diff --git a/src/storages/EventsDB.ts b/src/storages/EventsDB.ts index bedc1cac..b4dc0b9b 100644 --- a/src/storages/EventsDB.ts +++ b/src/storages/EventsDB.ts @@ -3,7 +3,7 @@ import { NPostgres, NPostgresSchema } from '@nostrify/db'; import { NIP50, NKinds, NostrEvent, NostrFilter, NSchema as n } from '@nostrify/nostrify'; import { Stickynotes } from '@soapbox/stickynotes'; -import { Kysely } from 'kysely'; +import { Kysely, SelectQueryBuilder } from 'kysely'; import { nip27 } from 'nostr-tools'; import { DittoTables } from '@/db/DittoTables.ts'; @@ -145,8 +145,36 @@ class EventsDB extends NPostgres { } protected getFilterQuery(trx: Kysely, filter: NostrFilter) { - const query = super.getFilterQuery(trx, filter); - return query; + if (filter.search) { + const tokens = NIP50.parseInput(filter.search); + + let query = super.getFilterQuery(trx, { + ...filter, + search: tokens.filter((t) => typeof t === 'string').join(' '), + }) as SelectQueryBuilder>; + + const data = tokens.filter((t) => typeof t === 'object').reduce( + (acc, t) => acc.set(t.key, t.value), + new Map(), + ); + + const domain = data.get('domain'); + const language = data.get('language'); + + if (domain) { + query = query + .innerJoin('pubkey_domains', 'nostr_events.pubkey', 'pubkey_domains.pubkey') + .where('pubkey_domains.domain', '=', domain); + } + + if (language) { + query = query.where('language', '=', language); + } + + return query; + } + + return super.getFilterQuery(trx, filter); } /** Get events for filters from the database. */ @@ -260,35 +288,6 @@ class EventsDB extends NPostgres { filters = structuredClone(filters); for (const filter of filters) { - if (filter.search) { - const tokens = NIP50.parseInput(filter.search); - - const domain = (tokens.find((t) => - typeof t === 'object' && t.key === 'domain' - ) as { key: 'domain'; value: string } | undefined)?.value; - - if (domain) { - const query = this.opts.kysely - .selectFrom('pubkey_domains') - .select('pubkey') - .where('domain', '=', domain); - - if (filter.authors) { - query.where('pubkey', 'in', filter.authors); - } - - const pubkeys = await query - .execute() - .then((rows) => - rows.map((row) => row.pubkey) - ); - - filter.authors = pubkeys; - } - - filter.search = tokens.filter((t) => typeof t === 'string').join(' '); - } - if (filter.kinds) { // Ephemeral events are not stored, so don't bother querying for them. // If this results in an empty kinds array, NDatabase will remove the filter before querying and return no results. From eede3909b11e4ae5abfaa16e319db29ec6d7264f Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 15 Sep 2024 13:05:54 -0500 Subject: [PATCH 69/82] Upgrade @nostrify/db to fix table joins --- deno.json | 2 +- deno.lock | 17 ++++++++++------- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/deno.json b/deno.json index 8edf0e53..ed8be157 100644 --- a/deno.json +++ b/deno.json @@ -31,7 +31,7 @@ "@isaacs/ttlcache": "npm:@isaacs/ttlcache@^1.4.1", "@lambdalisue/async": "jsr:@lambdalisue/async@^2.1.1", "@noble/secp256k1": "npm:@noble/secp256k1@^2.0.0", - "@nostrify/db": "jsr:@nostrify/db@^0.31.2", + "@nostrify/db": "jsr:@nostrify/db@^0.32.2", "@nostrify/nostrify": "jsr:@nostrify/nostrify@^0.30.1", "@scure/base": "npm:@scure/base@^1.1.6", "@sentry/deno": "https://deno.land/x/sentry@7.112.2/index.mjs", diff --git a/deno.lock b/deno.lock index 3a25d8d3..0a60494a 100644 --- a/deno.lock +++ b/deno.lock @@ -15,7 +15,7 @@ "jsr:@gleasonator/policy@0.5.2": "jsr:@gleasonator/policy@0.5.2", "jsr:@hono/hono@^4.4.6": "jsr:@hono/hono@4.5.11", "jsr:@lambdalisue/async@^2.1.1": "jsr:@lambdalisue/async@2.1.1", - "jsr:@nostrify/db@^0.31.2": "jsr:@nostrify/db@0.31.2", + "jsr:@nostrify/db@^0.32.2": "jsr:@nostrify/db@0.32.2", "jsr:@nostrify/nostrify@^0.22.1": "jsr:@nostrify/nostrify@0.22.5", "jsr:@nostrify/nostrify@^0.22.4": "jsr:@nostrify/nostrify@0.22.4", "jsr:@nostrify/nostrify@^0.22.5": "jsr:@nostrify/nostrify@0.22.5", @@ -47,7 +47,7 @@ "jsr:@std/fmt@0.213.1": "jsr:@std/fmt@0.213.1", "jsr:@std/fs@0.213.1": "jsr:@std/fs@0.213.1", "jsr:@std/fs@^0.229.3": "jsr:@std/fs@0.229.3", - "jsr:@std/internal@^1.0.0": "jsr:@std/internal@1.0.1", + "jsr:@std/internal@^1.0.0": "jsr:@std/internal@1.0.3", "jsr:@std/io@^0.223.0": "jsr:@std/io@0.223.0", "jsr:@std/io@^0.224": "jsr:@std/io@0.224.7", "jsr:@std/json@^0.223.0": "jsr:@std/json@0.223.0", @@ -198,11 +198,11 @@ "@lambdalisue/async@2.1.1": { "integrity": "1fc9bc6f4ed50215cd2f7217842b18cea80f81c25744f88f8c5eb4be5a1c9ab4" }, - "@nostrify/db@0.31.2": { - "integrity": "a906b64edbf84a6b482cd7c9f5df2d2237c4ec42589116097d99ceb41347b1f5", + "@nostrify/db@0.32.2": { + "integrity": "265fb41e9d5810b99f1003ce56c89e4b468e6d0c04e7b9d9e3126c4efd49c1c2", "dependencies": [ - "jsr:@nostrify/nostrify@^0.30.0", - "jsr:@nostrify/types@^0.30.0", + "jsr:@nostrify/nostrify@^0.31.0", + "jsr:@nostrify/types@^0.30.1", "npm:kysely@^0.27.3", "npm:nostr-tools@^2.7.0" ] @@ -382,6 +382,9 @@ "@std/internal@1.0.1": { "integrity": "6f8c7544d06a11dd256c8d6ba54b11ed870aac6c5aeafff499892662c57673e6" }, + "@std/internal@1.0.3": { + "integrity": "208e9b94a3d5649bd880e9ca38b885ab7651ab5b5303a56ed25de4755fb7b11e" + }, "@std/io@0.223.0": { "integrity": "2d8c3c2ab3a515619b90da2c6ff5ea7b75a94383259ef4d02116b228393f84f1", "dependencies": [ @@ -1994,7 +1997,7 @@ "jsr:@bradenmacdonald/s3-lite-client@^0.7.4", "jsr:@hono/hono@^4.4.6", "jsr:@lambdalisue/async@^2.1.1", - "jsr:@nostrify/db@^0.31.2", + "jsr:@nostrify/db@^0.32.2", "jsr:@nostrify/nostrify@^0.30.1", "jsr:@soapbox/kysely-pglite@^0.0.1", "jsr:@soapbox/stickynotes@^0.4.0", From f8902760cef6a66d5dd47abfdde4dcd5226703b2 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 15 Sep 2024 13:37:23 -0500 Subject: [PATCH 70/82] Make the language index a compound index --- src/db/migrations/032_add_language.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/db/migrations/032_add_language.ts b/src/db/migrations/032_add_language.ts index a0f828fe..77bfc37e 100644 --- a/src/db/migrations/032_add_language.ts +++ b/src/db/migrations/032_add_language.ts @@ -2,10 +2,14 @@ import { Kysely } from 'kysely'; export async function up(db: Kysely): Promise { await db.schema.alterTable('nostr_events').addColumn('language', 'char(2)').execute(); - await db.schema.createIndex('nostr_events_language_idx').on('nostr_events').column('language').execute(); + + await db.schema.createIndex('nostr_events_language_created_idx') + .on('nostr_events') + .columns(['language', 'created_at desc', 'id asc', 'kind']) + .execute(); } export async function down(db: Kysely): Promise { await db.schema.alterTable('nostr_events').dropColumn('language').execute(); - await db.schema.dropIndex('nostr_events_language_idx').execute(); + await db.schema.dropIndex('nostr_events_language_created_idx').execute(); } From d7ae3722c8e3a4557e997ed787e52b99c449b19b Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Sun, 15 Sep 2024 16:52:49 -0300 Subject: [PATCH 71/82] refactor: insert each event per iteration in for loop - db:populate-search --- scripts/db-populate-search.ts | 33 +++++++++++++++------------------ 1 file changed, 15 insertions(+), 18 deletions(-) diff --git a/scripts/db-populate-search.ts b/scripts/db-populate-search.ts index beec2d52..e34aaa75 100644 --- a/scripts/db-populate-search.ts +++ b/scripts/db-populate-search.ts @@ -1,12 +1,9 @@ import { NSchema as n } from '@nostrify/nostrify'; import { Storages } from '@/storages.ts'; -import { DittoTables } from '@/db/DittoTables.ts'; const store = await Storages.db(); const kysely = await Storages.kysely(); -const values: DittoTables['author_search'][] = []; - for await (const msg of store.req([{ kinds: [0] }])) { if (msg[0] === 'EVENT') { const { pubkey, content } = msg[2]; @@ -14,22 +11,22 @@ for await (const msg of store.req([{ kinds: [0] }])) { const { name, nip05 } = n.json().pipe(n.metadata()).catch({}).parse(content); const search = [name, nip05].filter(Boolean).join(' ').trim(); - values.push({ - pubkey: pubkey, - search, - }); + try { + await kysely.insertInto('author_search').values({ + pubkey, + search, + }).onConflict( + (oc) => + oc.column('pubkey') + .doUpdateSet((eb) => ({ search: eb.ref('excluded.search') })), + ) + .execute(); + } catch { + // do nothing + } + } else { + break; } } -try { - await kysely.insertInto('author_search').values(values).onConflict( - (oc) => - oc.column('pubkey') - .doUpdateSet((eb) => ({ search: eb.ref('excluded.search') })), - ) - .execute(); -} catch { - // do nothing -} - Deno.exit(); From 3b8a800cd2bb4bbb2659776a96b228169044b474 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Sun, 15 Sep 2024 17:27:56 -0300 Subject: [PATCH 72/82] refactor(search enchance): map over pubkeys instead of events --- src/controllers/api/accounts.ts | 15 +++++---------- src/controllers/api/search.ts | 15 +++++---------- 2 files changed, 10 insertions(+), 20 deletions(-) diff --git a/src/controllers/api/accounts.ts b/src/controllers/api/accounts.ts index a8aa5d55..4b684c33 100644 --- a/src/controllers/api/accounts.ts +++ b/src/controllers/api/accounts.ts @@ -137,20 +137,15 @@ const accountSearchController: AppController = async (c) => { const pubkeys = await getPubkeysBySearch(kysely, { q: query, limit }); - const events = event ? [event] : await store.query([{ kinds: [0], authors: pubkeys, limit }], { + let events = event ? [event] : await store.query([{ kinds: [0], authors: pubkeys, limit }], { signal, }); - const orderedEvents = events.map((event, index) => { - const pubkey = pubkeys[index]; + events = pubkeys.map((pubkey) => { + return events.find((event) => event.pubkey === pubkey); + }).filter((event) => event !== undefined); - const orderedEvent = events.find((e) => e.pubkey === pubkey); - if (orderedEvent) return orderedEvent; - - return event; - }); - - const accounts = await hydrateEvents({ events: orderedEvents, store, signal }).then( + const accounts = await hydrateEvents({ events, store, signal }).then( (events) => Promise.all( events.map((event) => renderAccount(event)), diff --git a/src/controllers/api/search.ts b/src/controllers/api/search.ts index d62c08e8..30bad8e9 100644 --- a/src/controllers/api/search.ts +++ b/src/controllers/api/search.ts @@ -104,21 +104,16 @@ async function searchEvents({ q, type, limit, account_id }: SearchQuery, signal: const store = await Storages.search(); - const events = await store.query([filter], { signal }) + let events = await store.query([filter], { signal }) .then((events) => hydrateEvents({ events, store, signal })); if (type !== 'accounts') return events; - const orderedEvents: NostrEvent[] = events.map((event, index) => { - const pubkey = pubkeys[index]; + events = pubkeys.map((pubkey) => { + return events.find((event) => event.pubkey === pubkey); + }).filter((event) => event !== undefined); - const orderedEvent = events.find((e) => e.pubkey === pubkey); - if (orderedEvent) return orderedEvent; - - return event; - }); - - return orderedEvents; + return events; } /** Get event kinds to search from `type` query param. */ From ed74b2464a56a51aa16a7a3f2db1353a4f610e8f Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Sun, 15 Sep 2024 17:42:26 -0300 Subject: [PATCH 73/82] refactor: write it like a normal if statement --- src/controllers/api/search.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/controllers/api/search.ts b/src/controllers/api/search.ts index 30bad8e9..e220413b 100644 --- a/src/controllers/api/search.ts +++ b/src/controllers/api/search.ts @@ -96,8 +96,11 @@ async function searchEvents({ q, type, limit, account_id }: SearchQuery, signal: pubkeys.push(...(await getPubkeysBySearch(kysely, { q, limit }))); - if (!filter?.authors) filter.authors = pubkeys; - else filter.authors.push(...pubkeys); + if (!filter?.authors) { + filter.authors = pubkeys; + } else { + filter.authors.push(...pubkeys); + } filter.search = undefined; } From c5711ea07173ffbbdab42319712df2497981b0fe Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Sun, 15 Sep 2024 17:42:58 -0300 Subject: [PATCH 74/82] refactor(accountSearchController): only reassign events if event is undefined --- src/controllers/api/accounts.ts | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/controllers/api/accounts.ts b/src/controllers/api/accounts.ts index 4b684c33..6e3cac5b 100644 --- a/src/controllers/api/accounts.ts +++ b/src/controllers/api/accounts.ts @@ -141,10 +141,11 @@ const accountSearchController: AppController = async (c) => { signal, }); - events = pubkeys.map((pubkey) => { - return events.find((event) => event.pubkey === pubkey); - }).filter((event) => event !== undefined); - + if (!event) { + events = pubkeys.map((pubkey) => { + return events.find((event) => event.pubkey === pubkey); + }).filter((event) => event !== undefined); + } const accounts = await hydrateEvents({ events, store, signal }).then( (events) => Promise.all( From dc69f21e0bc93c2befabe7a43361c626bb46e653 Mon Sep 17 00:00:00 2001 From: "P. Reis" Date: Sun, 15 Sep 2024 17:46:10 -0300 Subject: [PATCH 75/82] refactor: write map function in a cleaner way --- src/controllers/api/accounts.ts | 6 +++--- src/controllers/api/search.ts | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/controllers/api/accounts.ts b/src/controllers/api/accounts.ts index 6e3cac5b..c946b697 100644 --- a/src/controllers/api/accounts.ts +++ b/src/controllers/api/accounts.ts @@ -142,9 +142,9 @@ const accountSearchController: AppController = async (c) => { }); if (!event) { - events = pubkeys.map((pubkey) => { - return events.find((event) => event.pubkey === pubkey); - }).filter((event) => event !== undefined); + events = pubkeys + .map((pubkey) => events.find((event) => event.pubkey === pubkey)) + .filter((event) => !!event); } const accounts = await hydrateEvents({ events, store, signal }).then( (events) => diff --git a/src/controllers/api/search.ts b/src/controllers/api/search.ts index e220413b..01fb6665 100644 --- a/src/controllers/api/search.ts +++ b/src/controllers/api/search.ts @@ -112,9 +112,9 @@ async function searchEvents({ q, type, limit, account_id }: SearchQuery, signal: if (type !== 'accounts') return events; - events = pubkeys.map((pubkey) => { - return events.find((event) => event.pubkey === pubkey); - }).filter((event) => event !== undefined); + events = pubkeys + .map((pubkey) => events.find((event) => event.pubkey === pubkey)) + .filter((event) => !!event); return events; } From 642ecfd36fe4f43e017a02e4ecf0bcf2b0ea05f2 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 15 Sep 2024 17:26:02 -0500 Subject: [PATCH 76/82] Rename language migration to 033 --- src/db/migrations/{032_add_language.ts => 033_add_language.ts} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename src/db/migrations/{032_add_language.ts => 033_add_language.ts} (100%) diff --git a/src/db/migrations/032_add_language.ts b/src/db/migrations/033_add_language.ts similarity index 100% rename from src/db/migrations/032_add_language.ts rename to src/db/migrations/033_add_language.ts From 394021e485542ef414165a4a46095df0b01dab84 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 15 Sep 2024 17:38:16 -0500 Subject: [PATCH 77/82] Don't use semi-colons in migration --- src/db/migrations/032_add_author_search.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/db/migrations/032_add_author_search.ts b/src/db/migrations/032_add_author_search.ts index d5a93c06..4323c252 100644 --- a/src/db/migrations/032_add_author_search.ts +++ b/src/db/migrations/032_add_author_search.ts @@ -8,8 +8,8 @@ export async function up(db: Kysely): Promise { .ifNotExists() .execute(); - await sql`CREATE EXTENSION IF NOT EXISTS pg_trgm;`.execute(db); - await sql`CREATE INDEX author_search_search_idx ON author_search USING GIN (search gin_trgm_ops);`.execute(db); + await sql`CREATE EXTENSION IF NOT EXISTS pg_trgm`.execute(db); + await sql`CREATE INDEX author_search_search_idx ON author_search USING GIN (search gin_trgm_ops)`.execute(db); } export async function down(db: Kysely): Promise { From 8510f22d1bf6708dbbebcf836fdf4bf87170df85 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 15 Sep 2024 18:33:16 -0500 Subject: [PATCH 78/82] Only save the nip05 in author search if it's valid --- src/pipeline.ts | 61 +++++++++++++++++++++++-------------------------- 1 file changed, 28 insertions(+), 33 deletions(-) diff --git a/src/pipeline.ts b/src/pipeline.ts index 06fc462a..87249b97 100644 --- a/src/pipeline.ts +++ b/src/pipeline.ts @@ -140,48 +140,43 @@ async function parseMetadata(event: NostrEvent, signal: AbortSignal): Promise undefined) : undefined; // Populate author_search. try { - const search = [name, nip05].filter(Boolean).join(' ').trim(); + const search = result?.pubkey === event.pubkey ? [name, nip05].filter(Boolean).join(' ').trim() : name ?? ''; - await kysely.insertInto('author_search').values({ - pubkey: event.pubkey, - search, - }).onConflict( - (oc) => - oc.column('pubkey') - .doUpdateSet({ search }), - ) - .execute(); + if (search) { + await kysely.insertInto('author_search').values({ + pubkey: event.pubkey, + search, + }).onConflict( + (oc) => + oc.column('pubkey') + .doUpdateSet({ search }), + ) + .execute(); + } } catch { // do nothing } - if (!nip05) return; + if (nip05 && result && result.pubkey === event.pubkey) { + // Track pubkey domain. + try { + const { domain } = parseNip05(nip05); - // Fetch nip05. - const result = await nip05Cache.fetch(nip05, { signal }).catch(() => undefined); - if (!result) return; - - // Ensure pubkey matches event. - const { pubkey } = result; - if (pubkey !== event.pubkey) return; - - // Track pubkey domain. - try { - const { domain } = parseNip05(nip05); - - await sql` - INSERT INTO pubkey_domains (pubkey, domain, last_updated_at) - VALUES (${pubkey}, ${domain}, ${event.created_at}) - ON CONFLICT(pubkey) DO UPDATE SET - domain = excluded.domain, - last_updated_at = excluded.last_updated_at - WHERE excluded.last_updated_at > pubkey_domains.last_updated_at - `.execute(kysely); - } catch (_e) { - // do nothing + await sql` + INSERT INTO pubkey_domains (pubkey, domain, last_updated_at) + VALUES (${event.pubkey}, ${domain}, ${event.created_at}) + ON CONFLICT(pubkey) DO UPDATE SET + domain = excluded.domain, + last_updated_at = excluded.last_updated_at + WHERE excluded.last_updated_at > pubkey_domains.last_updated_at + `.execute(kysely); + } catch (_e) { + // do nothing + } } } From 5a98ba86bfe38fdb2f5ca0df3066afc269e72e4b Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 15 Sep 2024 18:58:34 -0500 Subject: [PATCH 79/82] Fix ambiguous postgres.js errors --- deno.json | 2 +- deno.lock | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/deno.json b/deno.json index fc0a6268..80c72b20 100644 --- a/deno.json +++ b/deno.json @@ -70,7 +70,7 @@ "nostr-tools": "npm:nostr-tools@2.5.1", "nostr-wasm": "npm:nostr-wasm@^0.1.0", "path-to-regexp": "npm:path-to-regexp@^7.1.0", - "postgres": "https://raw.githubusercontent.com/xyzshantaram/postgres.js/8a9bbce88b3f6425ecaacd99a80372338b157a53/deno/mod.js", + "postgres": "https://gitlab.com/soapbox-pub/postgres.js/-/raw/e79d7d2039446fbf7a37d4eca0d17e94a94b8b53/deno/mod.js", "prom-client": "npm:prom-client@^15.1.2", "question-deno": "https://raw.githubusercontent.com/ocpu/question-deno/10022b8e52555335aa510adb08b0a300df3cf904/mod.ts", "tldts": "npm:tldts@^6.0.14", diff --git a/deno.lock b/deno.lock index 0a60494a..657162fd 100644 --- a/deno.lock +++ b/deno.lock @@ -1943,6 +1943,18 @@ "https://gitlab.com/soapbox-pub/kysely-deno-postgres/-/raw/main/mod.ts": "662438fd3909984bb8cbaf3fd44d2121e949d11301baf21d6c3f057ccf9887de", "https://gitlab.com/soapbox-pub/kysely-deno-postgres/-/raw/main/src/PostgreSQLDriver.ts": "590c2fa248cff38e6e0f623050983039b5fde61e9c7131593d2922fb1f0eb921", "https://gitlab.com/soapbox-pub/kysely-deno-postgres/-/raw/main/src/PostgreSQLDriverDatabaseConnection.ts": "2158de426860bfd4f8e73afff0289bd40a11e273c8d883d4fd6474db01a9c2a7", + "https://gitlab.com/soapbox-pub/postgres.js/-/raw/e79d7d2039446fbf7a37d4eca0d17e94a94b8b53/deno/mod.js": "cb68f17d6d90df318934deccdb469d740be0888e7a597a9e7eea7100ce36a252", + "https://gitlab.com/soapbox-pub/postgres.js/-/raw/e79d7d2039446fbf7a37d4eca0d17e94a94b8b53/deno/polyfills.js": "318eb01f2b4cc33a46c59f3ddc11f22a56d6b1db8b7719b2ad7decee63a5bd47", + "https://gitlab.com/soapbox-pub/postgres.js/-/raw/e79d7d2039446fbf7a37d4eca0d17e94a94b8b53/deno/src/bytes.js": "f2de43bdc8fa5dc4b169f2c70d5d8b053a3dea8f85ef011d7b27dec69e14ebb7", + "https://gitlab.com/soapbox-pub/postgres.js/-/raw/e79d7d2039446fbf7a37d4eca0d17e94a94b8b53/deno/src/connection.js": "63bb06ad07cf802d295b35788261c34e82a80cec30b0dffafe05ccd74af3716f", + "https://gitlab.com/soapbox-pub/postgres.js/-/raw/e79d7d2039446fbf7a37d4eca0d17e94a94b8b53/deno/src/errors.js": "85cfbed9a5ab0db41ab8e97b806c881af29807dfe99bc656fdf1a18c1c13b6c6", + "https://gitlab.com/soapbox-pub/postgres.js/-/raw/e79d7d2039446fbf7a37d4eca0d17e94a94b8b53/deno/src/index.js": "4e8b09c7d0ce6e9eea386f59337867266498d5bb60ccd567d0bea5da03f6094d", + "https://gitlab.com/soapbox-pub/postgres.js/-/raw/e79d7d2039446fbf7a37d4eca0d17e94a94b8b53/deno/src/large.js": "f3e770cdb7cc695f7b50687b4c6c4b7252129515486ec8def98b7582ee7c54ef", + "https://gitlab.com/soapbox-pub/postgres.js/-/raw/e79d7d2039446fbf7a37d4eca0d17e94a94b8b53/deno/src/query.js": "67c45a5151032aa46b587abc15381fe4efd97c696e5c1b53082b8161309c4ee2", + "https://gitlab.com/soapbox-pub/postgres.js/-/raw/e79d7d2039446fbf7a37d4eca0d17e94a94b8b53/deno/src/queue.js": "709624843223ea842bf095f6934080f19f1a059a51cbbf82e9827f3bb1bf2ca7", + "https://gitlab.com/soapbox-pub/postgres.js/-/raw/e79d7d2039446fbf7a37d4eca0d17e94a94b8b53/deno/src/result.js": "001ff5e0c8d634674f483d07fbcd620a797e3101f842d6c20ca3ace936260465", + "https://gitlab.com/soapbox-pub/postgres.js/-/raw/e79d7d2039446fbf7a37d4eca0d17e94a94b8b53/deno/src/subscribe.js": "9e4d0c3e573a6048e77ee2f15abbd5bcd17da9ca85a78c914553472c6d6c169b", + "https://gitlab.com/soapbox-pub/postgres.js/-/raw/e79d7d2039446fbf7a37d4eca0d17e94a94b8b53/deno/src/types.js": "471f4a6c35412aa202a7c177c0a7e5a7c3bd225f01bbde67c947894c1b8bf6ed", "https://raw.githubusercontent.com/ocpu/question-deno/10022b8e52555335aa510adb08b0a300df3cf904/KeyCombo.ts": "a370b2dca76faa416d00e45479c8ce344971b5b86b44b4d0b213245c4bd2f8a3", "https://raw.githubusercontent.com/ocpu/question-deno/10022b8e52555335aa510adb08b0a300df3cf904/checkbox.ts": "e337ee7396aaefe6cc8c6349a445542fe7f0760311773369c9012b3fa278d21e", "https://raw.githubusercontent.com/ocpu/question-deno/10022b8e52555335aa510adb08b0a300df3cf904/config.ts": "a94a022c757f63ee7c410e29b97d3bfab1811889fb4483f56395cf376a911d1b", From a87497380a0fbbad4fb4f9de5ef6869d6bde3ba5 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 15 Sep 2024 19:22:14 -0500 Subject: [PATCH 80/82] Make pipeline query easier to look at --- src/pipeline.ts | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/src/pipeline.ts b/src/pipeline.ts index 87249b97..8ca7ae5f 100644 --- a/src/pipeline.ts +++ b/src/pipeline.ts @@ -147,14 +147,9 @@ async function parseMetadata(event: NostrEvent, signal: AbortSignal): Promise - oc.column('pubkey') - .doUpdateSet({ search }), - ) + await kysely.insertInto('author_search') + .values({ pubkey: event.pubkey, search }) + .onConflict((oc) => oc.column('pubkey').doUpdateSet({ search })) .execute(); } } catch { From 0dcb1965bed7f89175128b9354b561824b32af87 Mon Sep 17 00:00:00 2001 From: Siddharth Singh Date: Mon, 16 Sep 2024 05:52:51 +0530 Subject: [PATCH 81/82] Change setup.ts to use password input instead of plaintext --- scripts/setup.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/setup.ts b/scripts/setup.ts index 1365fec5..32376692 100644 --- a/scripts/setup.ts +++ b/scripts/setup.ts @@ -54,7 +54,7 @@ if (DATABASE_URL) { const host = await question('input', 'Postgres host', 'localhost'); const port = await question('input', 'Postgres port', '5432'); const user = await question('input', 'Postgres user', 'ditto'); - const password = await question('input', 'Postgres password', 'ditto'); + const password = await question('password', 'Postgres password', true); const database = await question('input', 'Postgres database', 'ditto'); vars.DATABASE_URL = `postgres://${user}:${password}@${host}:${port}/${database}`; } From 8e3ddaa0561e6c8826561032c9fbda49782274d2 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Mon, 16 Sep 2024 12:31:50 -0500 Subject: [PATCH 82/82] Add PGLITE_DEBUG environment variable --- src/config.ts | 4 ++++ src/db/DittoDB.ts | 2 +- src/db/DittoDatabase.ts | 1 + src/db/adapters/DittoPglite.ts | 9 ++++++--- src/storages.ts | 5 ++++- 5 files changed, 16 insertions(+), 5 deletions(-) diff --git a/src/config.ts b/src/config.ts index e174c81a..f007341f 100644 --- a/src/config.ts +++ b/src/config.ts @@ -77,6 +77,10 @@ class Conf { static get testDatabaseUrl(): string { return Deno.env.get('TEST_DATABASE_URL') ?? 'memory://'; } + /** PGlite debug level. 0 disables logging. */ + static get pgliteDebug(): 0 | 1 | 2 | 3 | 4 | 5 { + return Number(Deno.env.get('PGLITE_DEBUG') || 0) as 0 | 1 | 2 | 3 | 4 | 5; + } static db = { /** Database query timeout configurations. */ timeouts: { diff --git a/src/db/DittoDB.ts b/src/db/DittoDB.ts index 445c3da2..923a109d 100644 --- a/src/db/DittoDB.ts +++ b/src/db/DittoDB.ts @@ -16,7 +16,7 @@ export class DittoDB { switch (protocol) { case 'file:': case 'memory:': - return DittoPglite.create(databaseUrl); + return DittoPglite.create(databaseUrl, opts); case 'postgres:': case 'postgresql:': return DittoPostgres.create(databaseUrl, opts); diff --git a/src/db/DittoDatabase.ts b/src/db/DittoDatabase.ts index 530d9391..ec9a103d 100644 --- a/src/db/DittoDatabase.ts +++ b/src/db/DittoDatabase.ts @@ -10,4 +10,5 @@ export interface DittoDatabase { export interface DittoDatabaseOpts { poolSize?: number; + debug?: 0 | 1 | 2 | 3 | 4 | 5; } diff --git a/src/db/adapters/DittoPglite.ts b/src/db/adapters/DittoPglite.ts index 0e93075d..2455fc37 100644 --- a/src/db/adapters/DittoPglite.ts +++ b/src/db/adapters/DittoPglite.ts @@ -3,15 +3,18 @@ import { pg_trgm } from '@electric-sql/pglite/contrib/pg_trgm'; import { PgliteDialect } from '@soapbox/kysely-pglite'; import { Kysely } from 'kysely'; -import { DittoDatabase } from '@/db/DittoDatabase.ts'; +import { DittoDatabase, DittoDatabaseOpts } from '@/db/DittoDatabase.ts'; import { DittoTables } from '@/db/DittoTables.ts'; import { KyselyLogger } from '@/db/KyselyLogger.ts'; export class DittoPglite { - static create(databaseUrl: string): DittoDatabase { + static create(databaseUrl: string, opts?: DittoDatabaseOpts): DittoDatabase { const kysely = new Kysely({ dialect: new PgliteDialect({ - database: new PGlite(databaseUrl, { extensions: { pg_trgm } }), + database: new PGlite(databaseUrl, { + extensions: { pg_trgm }, + debug: opts?.debug, + }), }), log: KyselyLogger, }); diff --git a/src/storages.ts b/src/storages.ts index cbafd5aa..073b6135 100644 --- a/src/storages.ts +++ b/src/storages.ts @@ -21,7 +21,10 @@ export class Storages { public static async database(): Promise { if (!this._database) { this._database = (async () => { - const db = DittoDB.create(Conf.databaseUrl, { poolSize: Conf.pg.poolSize }); + const db = DittoDB.create(Conf.databaseUrl, { + poolSize: Conf.pg.poolSize, + debug: Conf.pgliteDebug, + }); await DittoDB.migrate(db.kysely); return db; })();